embulk 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 326eaaf38d4c941202486e516d30384f7e536abb
-  data.tar.gz: e017c2ee071139ebd4d1af63021fd08d7bb18226
+  metadata.gz: 69167ac2d89cb01cce7fe04d3380290604141720
+  data.tar.gz: cfcc577be3131bd9792d420425fd10dedf2df974
 SHA512:
-  metadata.gz: 95581d98106ba22022063f27f1c3a17f682b20bdc9709ea6e315847595634fce0471806d0abf2876f07f9c3e8bc30b60b379db655ceb892cca8c370ec4610ae2
-  data.tar.gz: 3e6e89b78cdc595aee32baa570d530c1d0acc45cb6865ed4416254737bf9583d157157cc1ff8e79ac5132c22b6fd25853892e3342eb689217c5144f3d6e08e3d
+  metadata.gz: ebe4adc87fd92f411e19db78e3ea58569c7902bf8e7f9dee97b797d5d7b3f502c056804d8e4d8a9989b6e61dc8e36476588fbf279d8232d48a4ea7842fa3411b
+  data.tar.gz: b2721835cd90a56cd9c97644a1af1429225542a8c0f1845014ce833e4f548dd48fed4511329f3f196da00d85783f8bb8bab73cf1e1f5effb6ff680fe5a13cacf

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.1'
+    version = '0.7.2'
 
     ext {
         jrubyVersion = '9.0.0.0'

data/embulk-core/src/main/java/org/embulk/config/ConfigException.java

@@ -4,6 +4,11 @@ public class ConfigException
         extends RuntimeException
         implements UserDataException
 {
+    protected ConfigException()
+    {
+        super();
+    }
+
     public ConfigException(String message)
     {
         super(message);

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java

@@ -1,10 +1,9 @@
 package org.embulk.spi.time;
 
-import org.embulk.config.UserDataException;
+import org.embulk.spi.DataException;
 
 public class TimestampParseException
-        extends Exception
-        implements UserDataException
+        extends DataException
 {
     public TimestampParseException(String message)
     {

data/embulk-docs/src/built-in.rst

@@ -180,6 +180,8 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default    |

data/embulk-docs/src/release.rst

@@ -4,8 +4,10 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.2
     release/release-0.7.1
     release/release-0.7.0
+    release/release-0.6.26
     release/release-0.6.25
     release/release-0.6.24
     release/release-0.6.23

data/embulk-docs/src/release/release-0.6.26.rst

@@ -0,0 +1,17 @@
+Release 0.6.26
+==================================
+
+Built-in plugins
+------------------
+
+* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
+
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
+
+Release Date
+------------------
+2015-08-20

data/embulk-docs/src/release/release-0.7.2.rst

@@ -0,0 +1,25 @@
+Release 0.7.2
+==================================
+
+Built-in plugins
+------------------
+
+* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
+
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
+* Fixed ``cannot be cast to org.jruby.RubyException`` when ConfigException is raised.
+
+
+General Changes
+------------------
+
+* Fixed ``embulk selfupdate`` subcommand (@cosmo0920++)
+
+
+Release Date
+------------------
+2015-08-20

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -90,6 +90,10 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         boolean getAllowExtraColumns();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
     }
 
     public static class QuoteCharacter
@@ -230,6 +234,7 @@ public class CsvParserPlugin
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -353,6 +358,9 @@ public class CsvParserPlugin
                     } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
+                        if (stopOnInvalidRecord) {
+                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
+                        }
                         log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
                         //exec.notice().skippedLine(skippedLine);
 

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -70,7 +70,9 @@ public class CsvTokenizer
             // recover lines of quoted value
             skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
             unreadLines.addAll(quotedValueLines);
-            unreadLines.add(line);
+            if (line != null) {
+                unreadLines.add(line);
+            }
             lineNumber -= quotedValueLines.size();
             quotedValueLines.clear();
         }

data/lib/embulk.rb

@@ -56,6 +56,8 @@ module Embulk
       raise "Embulk.setup works only with JRuby."
     end
 
+    require 'json'
+
     require_classpath
 
     systemConfigJson = system_config.merge({

data/lib/embulk/command/embulk_run.rb

@@ -173,7 +173,7 @@ examples:
 
     when :selfupdate
       op.on('-f', "Skip corruption check", TrueClass) do |b|
-        system[:force] = true
+        options[:force] = true
       end
       args = 0..0
 
@@ -253,7 +253,7 @@ examples:
 
     when :selfupdate
       require 'embulk/command/embulk_selfupdate'
-      Embulk.selfupdate(system)
+      Embulk.selfupdate(options)
 
     else
       require 'json'

data/lib/embulk/error.rb

@@ -1,17 +1,26 @@
 
 module Embulk
-  module UserDataError
-    include Java::Config::UserDataException
+  # ConfigError is not a ::StandardError but is a java.lang.RuntimeException.
+  # "rescue => e" can rescues ConfigError.
+  class ConfigError < Java::Config::ConfigException
+    def initialize(message=nil)
+      if message
+        super(message.to_s)
+      else
+        super()
+      end
+    end
   end
 
-  class ConfigError < StandardError
-    include UserDataError
-  end
-
-  class DataError < StandardError
-    include UserDataError
-  end
-
-  class PluginLoadError < StandardError
+  # DataError is not a ::StandardError but is a java.lang.RuntimeException.
+  # "rescue => e" can rescues DataError.
+  class DataError < Java::SPI::DataException
+    def initialize(message=nil)
+      if message
+        super(message.to_s)
+      else
+        super()
+      end
+    end
   end
 end

data/lib/embulk/exec.rb

@@ -0,0 +1,8 @@
+
+module Embulk
+  module Exec
+    def self.preview?
+      Java::Exec.isPreview
+    end
+  end
+end

data/lib/embulk/plugin.rb

@@ -15,6 +15,7 @@ module Embulk
   require 'embulk/guess_plugin'
   require 'embulk/executor_plugin'
   require 'embulk/java_plugin' if Embulk.java?
+  require 'embulk/exec' if Embulk.java?
 
   class PluginManager
     def initialize

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.1'
+  VERSION = '0.7.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -121,8 +121,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.1.jar
-- classpath/embulk-standards-0.7.1.jar
+- classpath/embulk-core-0.7.2.jar
+- classpath/embulk-standards-0.7.2.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -409,6 +409,7 @@ files:
 - embulk-docs/src/release/release-0.6.23.rst
 - embulk-docs/src/release/release-0.6.24.rst
 - embulk-docs/src/release/release-0.6.25.rst
+- embulk-docs/src/release/release-0.6.26.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -418,6 +419,7 @@ files:
 - embulk-docs/src/release/release-0.6.9.rst
 - embulk-docs/src/release/release-0.7.0.rst
 - embulk-docs/src/release/release-0.7.1.rst
+- embulk-docs/src/release/release-0.7.2.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -493,6 +495,7 @@ files:
 - lib/embulk/decoder_plugin.rb
 - lib/embulk/encoder_plugin.rb
 - lib/embulk/error.rb
+- lib/embulk/exec.rb
 - lib/embulk/executor_plugin.rb
 - lib/embulk/file_input.rb
 - lib/embulk/file_input_plugin.rb