embulk 0.8.6 → 0.8.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eaa5606c99e0324b6e5057569535386d5b9e3988
-  data.tar.gz: a55d68015412d824e5c77e310da4564488898590
+  metadata.gz: d0a538ad48c7719c664804156957e242d4d26db1
+  data.tar.gz: 2fb8a410c1d8a09e05f74f84797f8b9e619e9d13
 SHA512:
-  metadata.gz: 05ede9f4f6ec3578e198827ef7e26a1089c2271c3a00c5c506b9bf7b702ce1ee650c7bcf4a39f9c3c4ce877a84d8348426c728d6a34b50c5ca46a35660adb7ae
-  data.tar.gz: 8b580c8b98a182465a70a3088de6c283a6b3c80c334176f40e3b08af7b915caf0918484140532f599393e2ff302cced24e79d438b8bb7defe9c8067bfe2a807a
+  metadata.gz: 179de8013257589f33ad8c6dcc26d06e4acea07c48b448e66d96616c785665ffb3d43980e66d24a595d00c14429d3ba7053c71739c2ddbff5b4f6bca2000705a
+  data.tar.gz: 1aa03816f66d0d60fac7464c56d3f967a84d020e552c7e07c528c3a450ae8039d01b77192fdfb2b3ed4a2e8aada696ef33dee08ba000d9028f3166697f5d285a

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.6'
+    version = '0.8.7'
 
     ext {
         jrubyVersion = '9.0.5.0'
@@ -232,10 +232,10 @@ project(':embulk-cli') {
         append("${rootProject.projectDir}/COPYING")
     }
 
-    task classpath(type: Copy) {
+    task classpath(type: Copy, dependsOn: ['jar']) {
         File dest = file("${rootProject.projectDir}/classpath")
         doFirst { dest.deleteDir() }
-        from configurations.runtime
+        from configurations.runtime + files("${project.libsDir}/${project.name}-${project.version}.jar")
         into dest
     }
 }
@@ -300,7 +300,7 @@ task releaseCheck << {
     println "Ready. Run 'release' task."
 }
 
-task release(dependsOn: ["releaseCheck", "bintrayUpload", "rubyGemsUpload"]) << {
+task release(dependsOn: ["cli", "releaseCheck", "bintrayUpload", "rubyGemsUpload"]) << {
     println """
 Manual operations:
 

data/embulk-core/src/main/java/org/embulk/config/TaskSerDe.java

@@ -142,14 +142,18 @@ class TaskSerDe
             Map<String, Object> objects = new ConcurrentHashMap<String, Object>();
             HashMap<String, FieldEntry> unusedMappings = new HashMap<>(mappings);
 
-            JsonToken current;
-            current = jp.getCurrentToken();
+            String key;
+            JsonToken current = jp.getCurrentToken();
             if (current == JsonToken.START_OBJECT) {
                 current = jp.nextToken();
+                key = jp.getCurrentName();
             }
-            for (; current != JsonToken.END_OBJECT; current = jp.nextToken()) {
-                String key = jp.getCurrentName();
-                current = jp.nextToken();
+            else {
+                key = jp.nextFieldName();
+            }
+
+            for (; key != null; key = jp.nextFieldName()) {
+                JsonToken t = jp.nextToken(); // to get to value
                 FieldEntry field = mappings.get(key);
                 if (field == null) {
                     jp.skipChildren();

data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java

@@ -13,6 +13,7 @@ import org.embulk.config.Task;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.spi.FileInput;
+import static java.nio.charset.StandardCharsets.UTF_8;
 
 public class LineDecoder
         implements AutoCloseable, Iterable<String>
@@ -33,10 +34,12 @@ public class LineDecoder
 
     private final FileInputInputStream inputStream;
     private final BufferedReader reader;
+    private final Charset charset;
 
     public LineDecoder(FileInput in, DecoderTask task)
     {
-        CharsetDecoder decoder = task.getCharset()
+        this.charset = task.getCharset();
+        CharsetDecoder decoder = charset
             .newDecoder()
             .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
             .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
@@ -46,7 +49,43 @@ public class LineDecoder
 
     public boolean nextFile()
     {
-        return inputStream.nextFile();
+        boolean has = inputStream.nextFile();
+        if (has && charset.equals(UTF_8)) {
+            skipBom();
+        }
+        return has;
+    }
+
+    private void skipBom()
+    {
+        boolean skip = false;
+        try {
+            if (charset.equals(UTF_8)) {
+                reader.mark(3);
+                int firstChar = reader.read();
+                if (firstChar == 0xFEFF) {
+                    // skip BOM bytes
+                    skip = true;
+                }
+            }
+        }
+        catch (IOException ex) {
+        }
+        finally {
+            if (skip) {
+                // firstChar is skipped
+            }
+            else {
+                // rollback to the marked position
+                try {
+                    reader.reset();
+                }
+                catch (IOException ex) {
+                    // unexpected
+                    throw new RuntimeException(ex);
+                }
+            }
+        }
     }
 
     public String poll()

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.7
     release/release-0.8.6
     release/release-0.8.5
     release/release-0.8.4

data/embulk-docs/src/release/release-0.8.7.rst

@@ -0,0 +1,18 @@
+Release 0.8.7
+==================================
+
+General Changes
+------------------
+
+* LineDecoder skips UTF-8 BOM appropriately.
+
+* Timestamp guess uses ``%z`` instead of ``%Z``. Parsing behavior doesn't change but formatting behavior changes if output plugins use guessed configuration to format timestamp (this is unlikely because guessed configuration is most likely used by input plugins to parse timestamp).
+
+* Fix uninitialized constant MessagePack (@sonots++)
+
+* Fixed a possible busy loop at TaskDeserializer.
+
+
+Release Date
+------------------
+2016-03-04

data/lib/embulk/guess/time_format_guess.rb

@@ -107,11 +107,15 @@ module Embulk::Guess
               format << '%N'
             end
 
-          when :zone_off
-            format << '%z'
-
-          when :zone_abb
-            format << '%Z'
+          when :zone
+            case option
+            when :extended
+              format << '%:z'
+            else
+              # :simple, :abb
+              # don't use %Z even with :abb: https://github.com/jruby/jruby/issues/3702
+              format << '%z'
+            end
 
           else
             raise "Unknown part: #{@parts[i]}"
@@ -275,12 +279,16 @@ module Embulk::Guess
 
         if zm = /^#{ZONE}$/.match(rest)
           delimiters << (zm["zone_space"] || '')
+          parts << :zone
           if zm["zone_off"]
-            parts << :zone_off
+            if zm["zone_off"].include?(':')
+              part_options << :extended
+            else
+              part_options << :simple
+            end
           else
-            parts << :zone_abb
+            part_options << :abb
           end
-          part_options << nil
 
           return GuessMatch.new(delimiters, parts, part_options)
 
@@ -334,8 +342,16 @@ module Embulk::Guess
           format << "%d %b %Y"
           format << " %H:%M" if m['time']
           format << ":%S" if m['second']
-          format << " %z" if m['zone_off']
-          format << " %Z" if m['zone_abb']
+          if m['zone_off']
+            if m['zone_off'].include?(':')
+              format << " %:z"
+            else
+              format << " %z"
+            end
+          elsif m['zone_abb']
+            # don't use %Z: https://github.com/jruby/jruby/issues/3702
+            format << " %z" if m['zone_abb']
+          end
           SimpleMatch.new(format)
         else
           nil

data/lib/embulk/schema.rb

@@ -1,6 +1,7 @@
 module Embulk
 
   require 'embulk/column'
+  require 'msgpack'
 
   class Schema < Array
     def initialize(columns)

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.8.6'
+  VERSION = '0.8.7'
 end

data/test/guess/test_time_format_guess.rb

@@ -66,24 +66,25 @@ class TimeFormatGuessTest < ::Test::Unit::TestCase
     assert_guess "%Y-%m-%dT%H:%M:%S", "2007-04-06T13:47:30"
     assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30Z"
     assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+00"
-    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+00:00"
+    assert_guess "%Y-%m-%dT%H:%M:%S%:z", "2007-04-06T13:47:30+00:00"
     assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30+0000"
     assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-01"
-    assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-01:30"
+    assert_guess "%Y-%m-%dT%H:%M:%S%:z", "2007-04-06T13:47:30-01:30"
     assert_guess "%Y-%m-%dT%H:%M:%S%z", "2007-04-06T13:47:30-0130"
   end
 
   def test_format_rfc_822_2822
-    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 14:02:34 PST"
-    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 22:02:34 UT"
-    assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 22:02:34 GMT"
-    assert_guess     '%d %b %Y %H:%M:%S %Z',      "20 Feb 2015 22:02:34 GMT"
-    assert_guess     '%d %b %Y %H:%M %Z',         "20 Feb 2015 22:02 GMT"
-    assert_guess '%a, %d %b %Y %H:%M %Z',    "Fri, 20 Feb 2015 22:02 GMT"
+    # This test is disabled because of https://github.com/jruby/jruby/issues/3702
+    #assert_guess '%a, %d %b %Y %H:%M:%S %Z', "Fri, 20 Feb 2015 14:02:34 PST"
+    assert_guess '%a, %d %b %Y %H:%M:%S %z', "Fri, 20 Feb 2015 22:02:34 UT"
+    assert_guess '%a, %d %b %Y %H:%M:%S %z', "Fri, 20 Feb 2015 22:02:34 GMT"
+    assert_guess     '%d %b %Y %H:%M:%S %z',      "20 Feb 2015 22:02:34 GMT"
+    assert_guess     '%d %b %Y %H:%M %z',         "20 Feb 2015 22:02 GMT"
+    assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 GMT"
     assert_guess     '%d %b %Y',                  "20 Feb 2015"
     assert_guess '%a, %d %b %Y',             "Fri, 20 Feb 2015"
     assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +0000"
-    assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +00:00"
+    assert_guess '%a, %d %b %Y %H:%M %:z',   "Fri, 20 Feb 2015 22:02 +00:00"
     assert_guess '%a, %d %b %Y %H:%M %z',    "Fri, 20 Feb 2015 22:02 +00"
   end
 
@@ -97,6 +98,7 @@ class TimeFormatGuessTest < ::Test::Unit::TestCase
 
   def test_format_merge_frequency
     assert_guess_partial 2, "%Y-%m-%d %H:%M:%S", ["2014-01-01", "2014-01-01 00:00:00", "2014-01-01 00:00:00"]
+    assert_guess_partial 3, "%Y-%m-%d %H:%M:%S %z", ["2014-01-01 00:00:00 +0000", "2014-01-01 00:00:00 +0000", "2014-01-01 00:00:00 +00:00"]
   end
 
   def test_format_merge_dmy

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.6
+  version: 0.8.7
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-24 00:00:00.000000000 Z
+date: 2016-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -108,8 +108,9 @@ files:
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.8.6.jar
-- classpath/embulk-standards-0.8.6.jar
+- classpath/embulk-cli-0.8.7.jar
+- classpath/embulk-core-0.8.7.jar
+- classpath/embulk-standards-0.8.7.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -426,6 +427,7 @@ files:
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst
 - embulk-docs/src/release/release-0.8.6.rst
+- embulk-docs/src/release/release-0.8.7.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java