embulk 0.7.1-java → 0.7.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4caef0e11a979edc6d12dc23d1bc89c4796cf869
-  data.tar.gz: 83ca9285638a2de602cae9e21341e8e63ffdfdef
+  metadata.gz: f134c6937fe0926c4f18a31c4bc43b0180673bfd
+  data.tar.gz: 0075a441e60c60291e38a607565036579286e9db
 SHA512:
-  metadata.gz: 9a5a51d2cb2ebeba9ec15892cd70e12595813920906f23478d4ebb88baaf7b497b2bc5a3d6e8faaa08c0e85ff455f158345a59246d0180128b08065d2e956d04
-  data.tar.gz: c3409ea3f3c3534c6e70d9b25bbc7ed6a0b3c30c73bddc719d576eea5698246cfec650b7716d373782a3a1befaaf4021e49d24826922abadf82948524a2926ad
+  metadata.gz: 1f8a1361a31eaf2a44c27bce626ac0a408b7d2f0f282614807e75e90ad8fb4791fb27ff5673be5b952559b0220147ed68b48e2f9a242aaf01ffab6dfa998dc0f
+  data.tar.gz: 4a5532254234c25eaeadde47e7c17aed6771551f0aa0030bcb134f3d10fb7e7d45f882529438a899342327fa539e508169fdff889cd9911831ff5b557a58cf8b

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.1'
+    version = '0.7.2'
 
     ext {
         jrubyVersion = '9.0.0.0'

data/embulk-core/src/main/java/org/embulk/config/ConfigException.java

@@ -4,6 +4,11 @@ public class ConfigException
         extends RuntimeException
         implements UserDataException
 {
+    protected ConfigException()
+    {
+        super();
+    }
+
     public ConfigException(String message)
     {
         super(message);

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParseException.java

@@ -1,10 +1,9 @@
 package org.embulk.spi.time;
 
-import org.embulk.config.UserDataException;
+import org.embulk.spi.DataException;
 
 public class TimestampParseException
-        extends Exception
-        implements UserDataException
+        extends DataException
 {
     public TimestampParseException(String message)
     {

data/embulk-docs/src/built-in.rst

@@ -180,6 +180,8 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default    |

data/embulk-docs/src/release.rst

@@ -4,8 +4,10 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.2
     release/release-0.7.1
     release/release-0.7.0
+    release/release-0.6.26
     release/release-0.6.25
     release/release-0.6.24
     release/release-0.6.23

data/embulk-docs/src/release/release-0.6.26.rst

@@ -0,0 +1,17 @@
+Release 0.6.26
+==================================
+
+Built-in plugins
+------------------
+
+* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
+
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
+
+Release Date
+------------------
+2015-08-20

data/embulk-docs/src/release/release-0.7.2.rst

@@ -0,0 +1,25 @@
+Release 0.7.2
+==================================
+
+Built-in plugins
+------------------
+
+* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
+
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
+* Fixed ``cannot be cast to org.jruby.RubyException`` when ConfigException is raised.
+
+
+General Changes
+------------------
+
+* Fixed ``embulk selfupdate`` subcommand (@cosmo0920++)
+
+
+Release Date
+------------------
+2015-08-20

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -90,6 +90,10 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         boolean getAllowExtraColumns();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
     }
 
     public static class QuoteCharacter
@@ -230,6 +234,7 @@ public class CsvParserPlugin
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -353,6 +358,9 @@ public class CsvParserPlugin
                     } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
+                        if (stopOnInvalidRecord) {
+                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
+                        }
                         log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
                         //exec.notice().skippedLine(skippedLine);
 

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -70,7 +70,9 @@ public class CsvTokenizer
             // recover lines of quoted value
             skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
             unreadLines.addAll(quotedValueLines);
-            unreadLines.add(line);
+            if (line != null) {
+                unreadLines.add(line);
+            }
             lineNumber -= quotedValueLines.size();
             quotedValueLines.clear();
         }

data/lib/embulk.rb

@@ -56,6 +56,8 @@ module Embulk
       raise "Embulk.setup works only with JRuby."
     end
 
+    require 'json'
+
     require_classpath
 
     systemConfigJson = system_config.merge({

data/lib/embulk/command/embulk_run.rb

@@ -173,7 +173,7 @@ examples:
 
     when :selfupdate
       op.on('-f', "Skip corruption check", TrueClass) do |b|
-        system[:force] = true
+        options[:force] = true
       end
       args = 0..0
 
@@ -253,7 +253,7 @@ examples:
 
     when :selfupdate
       require 'embulk/command/embulk_selfupdate'
-      Embulk.selfupdate(system)
+      Embulk.selfupdate(options)
 
     else
       require 'json'

data/lib/embulk/error.rb

@@ -1,17 +1,26 @@
 
 module Embulk
-  module UserDataError
-    include Java::Config::UserDataException
+  # ConfigError is not a ::StandardError but is a java.lang.RuntimeException.
+  # "rescue => e" can rescues ConfigError.
+  class ConfigError < Java::Config::ConfigException
+    def initialize(message=nil)
+      if message
+        super(message.to_s)
+      else
+        super()
+      end
+    end
   end
 
-  class ConfigError < StandardError
-    include UserDataError
-  end
-
-  class DataError < StandardError
-    include UserDataError
-  end
-
-  class PluginLoadError < StandardError
+  # DataError is not a ::StandardError but is a java.lang.RuntimeException.
+  # "rescue => e" can rescues DataError.
+  class DataError < Java::SPI::DataException
+    def initialize(message=nil)
+      if message
+        super(message.to_s)
+      else
+        super()
+      end
+    end
   end
 end

data/lib/embulk/exec.rb

@@ -0,0 +1,8 @@
+
+module Embulk
+  module Exec
+    def self.preview?
+      Java::Exec.isPreview
+    end
+  end
+end

data/lib/embulk/plugin.rb

@@ -15,6 +15,7 @@ module Embulk
   require 'embulk/guess_plugin'
   require 'embulk/executor_plugin'
   require 'embulk/java_plugin' if Embulk.java?
+  require 'embulk/exec' if Embulk.java?
 
   class PluginManager
     def initialize

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.1'
+  VERSION = '0.7.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: java
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -133,8 +133,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.1.jar
-- classpath/embulk-standards-0.7.1.jar
+- classpath/embulk-core-0.7.2.jar
+- classpath/embulk-standards-0.7.2.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -421,6 +421,7 @@ files:
 - embulk-docs/src/release/release-0.6.23.rst
 - embulk-docs/src/release/release-0.6.24.rst
 - embulk-docs/src/release/release-0.6.25.rst
+- embulk-docs/src/release/release-0.6.26.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -430,6 +431,7 @@ files:
 - embulk-docs/src/release/release-0.6.9.rst
 - embulk-docs/src/release/release-0.7.0.rst
 - embulk-docs/src/release/release-0.7.1.rst
+- embulk-docs/src/release/release-0.7.2.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -505,6 +507,7 @@ files:
 - lib/embulk/decoder_plugin.rb
 - lib/embulk/encoder_plugin.rb
 - lib/embulk/error.rb
+- lib/embulk/exec.rb
 - lib/embulk/executor_plugin.rb
 - lib/embulk/file_input.rb
 - lib/embulk/file_input_plugin.rb