embulk 0.5.2 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c731d85a93052e3fa0e40a49c048f365e150eace
-  data.tar.gz: 5266d8207396e64c995d8fd6e2b9e60cc228041c
+  metadata.gz: 0cd34264ca6a673f948b77e42cb3049e184a4f40
+  data.tar.gz: 8d56e898df4411110bc5c42c6817803d93a732fe
 SHA512:
-  metadata.gz: 658da0f21342555a4d267bfa5bd7ee048dddd875fdc45dff82d22fb7f00e19d1f735afed15549adf94f135ccae3ed327636a86b37a5b6a252de0821f4049a5a9
-  data.tar.gz: 7888c38856e8e3816eb819bc1b12f33852634882c7e40b68496ad5700cf7c1a202174e097cfeb1589f6847752e768d40fdfc9ef5d5465b698708882488c9aeeb
+  metadata.gz: 13ad1c043fff795f33199985f955d67238d4d54297516cd3d3ac8abcd1ea90e9cc43569e2745918c6090ce4373ae01a92f2cd26f35fddd37a93aa5d70c8be273
+  data.tar.gz: f02d113e3eddb344f9080d9dc5163c61e6718455b4d404607eb0a1bd62ac94da547697dd2be5796c6a44404a92ef01c845553f7da7a672b005dc434c5b3c5c13

data/README.md

@@ -30,7 +30,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 Following 4 commands install embulk to your home directory:
 
 ```
-curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.2.jar
+curl --create-dirs -o ~/.embulk/bin/embulk -L https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar
 chmod +x ~/.embulk/bin/embulk
 echo 'export PATH="$HOME/.embulk/bin:$PATH"' >> ~/.bashrc
 source ~/.bashrc
@@ -45,7 +45,7 @@ Embulk is a Java application. Please make sure that you installed [Java](http://
 You can assume the jar file is a .bat file.
 
 ```
-PowerShell -Command "& {Invoke-WebRequest https://bintray.com/artifact/download/embulk/maven/embulk-0.5.2.jar -OutFile embulk.bat}"
+PowerShell -Command "& {Invoke-WebRequest https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar -OutFile embulk.bat}"
 ```
 
 Next step: [Trying examples](#trying-examples)

data/build.gradle

@@ -12,7 +12,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.5.2'
+    version = '0.5.3'
 
     apply plugin: 'java'
     apply plugin: 'maven-publish'
@@ -105,7 +105,11 @@ subprojects {
             }
         }
 
-        // add javadoc/source jar tasks as artifacts to be released
+        // add tests/javadoc/source jar tasks as artifacts to be released
+        task testsJar(type: Jar, dependsOn: classes) {
+            classifier = 'tests'
+            from sourceSets.test.output
+        }
         task sourcesJar(type: Jar, dependsOn: classes) {
             classifier = 'sources'
             from sourceSets.main.allSource
@@ -115,7 +119,7 @@ subprojects {
             from javadoc.destinationDir
         }
         artifacts {
-            archives sourcesJar, javadocJar
+            archives testsJar, sourcesJar, javadocJar
         }
     }
 
@@ -124,6 +128,7 @@ subprojects {
             if (release_projects.contains(project)) {
                 bintrayMavenRelease(MavenPublication) {
                     from components.java
+                    artifact testsJar
                     artifact sourcesJar
                     artifact javadocJar
                 }

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -51,6 +51,7 @@ public class SamplingParserPlugin
         // override in.parser.type so that FileInputRunner creates GuessParserPlugin
         ConfigSource samplingInputConfig = inputConfig.deepCopy();
         samplingInputConfig.getNestedOrSetEmpty("parser").set("type", "system_sampling");
+        samplingInputConfig.set("decoders", null);
 
         try {
             runner.transaction(samplingInputConfig, new InputPlugin.Control() {
@@ -78,23 +79,23 @@ public class SamplingParserPlugin
 
     private static Buffer getSample(FileInput fileInput, int maxSampleSize)
     {
+        if (!fileInput.nextFile()) {
+            // no input files
+            return Buffer.EMPTY;
+        }
+
         Buffer sample = Buffer.allocate(maxSampleSize);
         int sampleSize = 0;
 
-        while (fileInput.nextFile()) {
-            for (Buffer buffer : each(fileInput)) {
-                if (sampleSize >= maxSampleSize) {
-                    // skip remaining all buffers so that FileInputPlugin.runInput doesn't
-                    // throw exceptions at channel.join()
-                } else {
-                    int size = Math.min(buffer.limit(), sample.capacity() - sampleSize);
-                    sample.setBytes(sampleSize, buffer, 0, size);
-                    sampleSize += size;
-                }
-                buffer.release();
+        for (Buffer buffer : each(fileInput)) {
+            int size = Math.min(buffer.limit(), sample.capacity() - sampleSize);
+            sample.setBytes(sampleSize, buffer, 0, size);
+            sampleSize += size;
+            buffer.release();
+            if (sampleSize >= maxSampleSize) {
+                break;
             }
         }
-
         sample.limit(sampleSize);
         return sample;
     }

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -56,7 +56,7 @@ You can find the latest embulk binary from the `releases <https://bintray.com/em
 
 .. code-block:: console
 
-    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.2.jar -O /usr/local/bin/embulk
+    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.3.jar -O /usr/local/bin/embulk
     $ sudo chmod +x /usr/local/bin/embulk
 
 Step 2. Install Elasticsearch plugin

data/embulk-docs/src/release.rst

@@ -24,4 +24,5 @@ Release Notes
     release/release-0.5.0
     release/release-0.5.1
     release/release-0.5.2
+    release/release-0.5.3
 

data/embulk-docs/src/release/release-0.5.3.rst

@@ -0,0 +1,22 @@
+Release 0.5.3
+==================================
+
+Built-in plugins
+------------------
+
+* ``guess`` guesses boolean types (@hata++)
+
+
+General Changes
+------------------
+
+* ``guess`` stops reading the file after first 32KB and ignores remaining data. Execution time of ``guess`` improves significantly especially when the first file is large.
+
+* ``guess`` ignores decoders when it reads sample data. This change fixes the problem where guess fails if config file includes ``decoder-gzip`` (@hata++).
+
+* Releases ``embulk-core-VERSION-tests.jar`` in addition to ``embulk-core-VERSION-sources.jar``.
+
+
+Release Date
+------------------
+2015-03-17

data/lib/embulk/command/embulk_run.rb

@@ -58,7 +58,7 @@ module Embulk
 
     when :run
       op.banner = "Usage: run <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
@@ -77,7 +77,7 @@ module Embulk
 
     when :cleanup
       op.banner = "Usage: cleanup <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
@@ -93,7 +93,7 @@ module Embulk
 
     when :preview
       op.banner = "Usage: preview <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
@@ -109,7 +109,7 @@ module Embulk
 
     when :guess
       op.banner = "Usage: guess <partial-config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, or trace)') do |level|
+      op.on('-l', '--log-level LEVEL', 'Log level (fatal, error, warn, info, debug or trace)') do |level|
         options[:logLevel] = level
       end
       op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|

data/lib/embulk/guess/schema_guess.rb

@@ -45,7 +45,7 @@ module Embulk::Guess
       private
 
       def guess_type(str)
-        if TRUE_STRINGS[str]
+        if TRUE_STRINGS[str] || FALSE_STRINGS[str]
           return "boolean"
         end
 
@@ -78,7 +78,7 @@ module Embulk::Guess
       end
 
       # taken from CsvParserPlugin.TRUE_STRINGS
-      TRUE_STRINGS = Hash[*%w[
+      TRUE_STRINGS = Hash[%w[
         true True TRUE
         yes Yes YES
         y Y
@@ -86,6 +86,15 @@ module Embulk::Guess
         1
       ].map {|k| [k, true] }]
 
+      # When matching to false string, then retrun 'true'
+      FALSE_STRINGS = Hash[%w[
+        false False FALSE
+        no No NO
+        n N
+        off Off OFF
+        0
+      ].map {|k| [k, true] }]
+
       TYPE_COALESCE = Hash[{
         long: :double,
         boolean: :long,

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.5.2'
+  VERSION = '0.5.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-11 00:00:00.000000000 Z
+date: 2015-03-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -274,6 +274,7 @@ files:
 - embulk-docs/src/release/release-0.5.0.rst
 - embulk-docs/src/release/release-0.5.1.rst
 - embulk-docs/src/release/release-0.5.2.rst
+- embulk-docs/src/release/release-0.5.3.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -378,8 +379,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.5.2.jar
-- classpath/embulk-standards-0.5.2.jar
+- classpath/embulk-core-0.5.3.jar
+- classpath/embulk-standards-0.5.3.jar
 - classpath/guava-18.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar