embulk 0.6.25 → 0.6.26

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5b13b5952b7f5c81cb0f9568c89aa67d66348699
-  data.tar.gz: c620fbe5b71f700d616c69f560a20dabc37de225
+  metadata.gz: 99bafeaaf1880afd940f1fbc79d93a932755a25d
+  data.tar.gz: 47375952c1fe8131eee784eb0f99bc27768127b3
 SHA512:
-  metadata.gz: 219495213f74635e2e0cb443fa0f070b5f94db4ca76826c99358c24c7d62447960de937c6654c139a0483fcb7c103e835fb5afcb9018ccef4050c739a63f5f0d
-  data.tar.gz: 3e3dafdf217da343592b766e6b2c986281d172f429afaf9d3ce0aee6ffb1d2a22aabb15f3a6bca93812c0149cd6291e31c74dd15befd88d51ca81b1beaf869c2
+  metadata.gz: 7f80c21126f7614b58b88fd3b5427bc1d8ca775c1ee1ccf5c00e887d54e199f215d4606df1c3cc14d670e973dfec362cbc505cf7e3e793a7978425dfb9f01e5f
+  data.tar.gz: 0c9261acf5b853b5f75efeaf13b3e9eb14f91c54f1d4b29b91721958427e3bb587820a0000679d79ba2df73613bdd872b04696a731b0f3814e544431f8074731

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.25'
+    version = '0.6.26'
 
     ext {
         jrubyVersion = '1.7.21'

data/embulk-docs/src/built-in.rst

@@ -181,6 +181,8 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default  |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default     |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default    |

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.26
     release/release-0.6.25
     release/release-0.6.24
     release/release-0.6.23

data/embulk-docs/src/release/release-0.6.26.rst

@@ -0,0 +1,17 @@
+Release 0.6.26
+==================================
+
+Built-in plugins
+------------------
+
+* Added ``stop_on_invalid_record`` option to ``parser-csv`` plugin. This option stops bulkloading if it finds a broken record such as invalid timestamp or invalid integer format rathar than skipping it.
+
+
+Ruby Plugin API
+------------------
+
+* Added ``Embulk::Exec.preview?`` which returns true if plugins are running in preview.
+
+Release Date
+------------------
+2015-08-20

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -91,6 +91,10 @@ public class CsvParserPlugin
         @Config("allow_extra_columns")
         @ConfigDefault("false")
         boolean getAllowExtraColumns();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
     }
 
     public static class QuoteCharacter
@@ -231,6 +235,7 @@ public class CsvParserPlugin
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
         int skipHeaderLines = task.getSkipHeaderLines();
 
         try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
@@ -354,6 +359,9 @@ public class CsvParserPlugin
                     } catch (CsvTokenizer.InvalidFormatException | CsvRecordValidateException e) {
                         long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
+                        if (stopOnInvalidRecord) {
+                            throw new ConfigException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
+                        }
                         log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
                         //exec.notice().skippedLine(skippedLine);
 

data/lib/embulk/exec.rb

@@ -0,0 +1,8 @@
+
+module Embulk
+  module Exec
+    def self.preview?
+      Java::Exec.isPreview
+    end
+  end
+end

data/lib/embulk/plugin.rb

@@ -15,6 +15,7 @@ module Embulk
   require 'embulk/guess_plugin'
   require 'embulk/executor_plugin'
   require 'embulk/java_plugin' if Embulk.java?
+  require 'embulk/exec' if Embulk.java?
 
   class PluginManager
     def initialize

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.25'
+  VERSION = '0.6.26'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.25
+  version: 0.6.26
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-16 00:00:00.000000000 Z
+date: 2015-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -104,8 +104,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.25.jar
-- classpath/embulk-standards-0.6.25.jar
+- classpath/embulk-core-0.6.26.jar
+- classpath/embulk-standards-0.6.26.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -386,6 +386,7 @@ files:
 - embulk-docs/src/release/release-0.6.23.rst
 - embulk-docs/src/release/release-0.6.24.rst
 - embulk-docs/src/release/release-0.6.25.rst
+- embulk-docs/src/release/release-0.6.26.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -465,6 +466,7 @@ files:
 - lib/embulk/decoder_plugin.rb
 - lib/embulk/encoder_plugin.rb
 - lib/embulk/error.rb
+- lib/embulk/exec.rb
 - lib/embulk/executor_plugin.rb
 - lib/embulk/file_input.rb
 - lib/embulk/file_input_plugin.rb