embulk 0.8.36-java → 0.8.37-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b186e9bc56699074dd4f996f4cea60fc4fa1f162dd04e5f5d63b85c503ad2f75
-  data.tar.gz: 83cb2e0da0505bfd6b7c77e76474de047421e912f16473079c324015dfde73b7
+  metadata.gz: 21524bac0f5a7148329477cdc4e9fd2989afd0ca2aef6844e8a58edcf95299c1
+  data.tar.gz: 8eb586244ac63524b85e72d25e6126935d36431d2a5775fcabe92ab7d5a2f0a5
 SHA512:
-  metadata.gz: 0e46da97361ff6202e80b453a4812b86cf23aff9ebd31c01938d053edddd76a85a7d1fc7abef81f4d281b51d3f5a703f7abf5cf6061794212ef024d072cff7cd
-  data.tar.gz: 402f2923df2fe35ac2b5029fd6b2058e41f22f0393a08e0eb215457ab405deae60eceb97c0d2df0dcf5e7b2c00cc7fcc9dfd243bad56477fe1713833edf1153c
+  metadata.gz: f78664ea4c04560fb307dd8c68cb3cd25dfae6edd7ea832655b5bcff604b9a5de9f447fd49f3ada2373fdfbd6afc9ef2ddc9ba98c169c8fa4fa90324313f14da
+  data.tar.gz: e39f5a6ea6d9baf2a7eb6fdc9eca6747fd208e3ff0813830ee4f28b6f548c6eb8a79ccf2b6b533a02092a738edadb9bdb52225ac92f858639e7503d2af945297

data/build.gradle

@@ -24,7 +24,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.36'
+    version = '0.8.37'
 
     ext {
         jrubyVersion = '9.1.13.0'
@@ -276,10 +276,14 @@ project(':embulk-cli') {
     }
 }
 
-task rubyTest(type: JRubyExec) {
-    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '--debug', './test/run-test.rb'
+task rubyTestVanilla(type: JRubyExec, dependsOn: "classpath") {
+    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '--debug', './test/vanilla/run-test.rb'
+}
+task rubyTestMonkeyStrptime(type: JRubyExec, dependsOn: "classpath") {
+    jrubyArgs '-Ilib', '-Itest', '-rtest/unit', '--debug', './test/monkey_strptime/run-test.rb'
+}
+task rubyTest(dependsOn: ["rubyTestVanilla", "rubyTestMonkeyStrptime"]) {
 }
-rubyTest.dependsOn('classpath')
 
 //
 // gem task

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -57,6 +57,12 @@ public class TimestampFormat
 
     public static DateTimeZone parseDateTimeZone(String s)
     {
+        final int rubyStyleTimeOffsetInSecond = TimeZoneConverter.dateZoneToDiff(s);
+
+        if (rubyStyleTimeOffsetInSecond != Integer.MIN_VALUE) {
+            return DateTimeZone.forOffsetMillis(rubyStyleTimeOffsetInSecond * 1000);
+        }
+
         if(s.startsWith("+") || s.startsWith("-")) {
             return DateTimeZone.forID(s);
 

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -187,7 +187,7 @@ public class TimestampParser
             // TODO cache parsed zone?
             timeZone = parseDateTimeZone(zone);
             if (timeZone == null) {
-                throw new TimestampParseException("Invalid time zone name '" + text + "'");
+                throw new TimestampParseException("Invalid time zone name '" + zone + "' in '" + text + "'");
             }
         }
         else {
@@ -201,7 +201,24 @@ public class TimestampParser
     public LocalTime createLocalTimeFromFormatBag(FormatBag bag)
     {
         final long secFractionNsec;
-        if (FormatBag.has(bag.getSecFraction())) {
+        if (FormatBag.has(bag.getSecondsSize())) {  // %Q is specified.
+            // Fractions by %Q are prioritized over fractions by %N.
+            // irb(main):002:0> Time.strptime("123456789 12.345", "%Q %S.%N").nsec
+            // => 789000000
+            // irb(main):003:0> Time.strptime("12.345 123456789", "%S.%N %Q").nsec
+            // => 789000000
+            // irb(main):004:0> Time.strptime("12.345", "%S.%N").nsec
+            // => 345000000
+            if (bag.hasSeconds()) {
+                secFractionNsec = (bag.getSeconds() % (int)Math.pow(10, bag.getSecondsSize())) *
+                                  (int)Math.pow(10, 9 - bag.getSecondsSize());
+            }
+            else {
+                // It must not happen -- |seconds| is always set if |secondsSize| is set.
+                secFractionNsec = 0;
+            }
+        }
+        else if (FormatBag.has(bag.getSecFraction())) {
             secFractionNsec = bag.getSecFraction() * (int)Math.pow(10, 9 - bag.getSecFractionSize());
         }
         else {
@@ -228,11 +245,16 @@ public class TimestampParser
 
             // set up with min this and then add to allow rolling over
             DateTime dt = new DateTime(year, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
-            if (FormatBag.has(bag.getMon())) {
-                dt = dt.plusMonths(bag.getMon() - 1);
+            if (FormatBag.has(bag.getYDay())) {  // yday is more prioritized than mon/mday in Ruby's strptime.
+                dt = dt.plusDays(bag.getYDay() - 1);
             }
-            if (FormatBag.has(bag.getMDay())) {
-                dt = dt.plusDays(bag.getMDay() - 1);
+            else {
+                if (FormatBag.has(bag.getMon())) {
+                    dt = dt.plusMonths(bag.getMon() - 1);
+                }
+                if (FormatBag.has(bag.getMDay())) {
+                    dt = dt.plusDays(bag.getMDay() - 1);
+                }
             }
             if (FormatBag.has(bag.getHour())) {
                 dt = dt.plusHours(bag.getHour());
@@ -241,7 +263,12 @@ public class TimestampParser
                 dt = dt.plusMinutes(bag.getMin());
             }
             if (FormatBag.has(bag.getSec())) {
-                dt = dt.plusSeconds(bag.getSec());
+                if (bag.getSec() == 60) {  // Leap seconds are considered as 59 when Ruby converts them to epochs.
+                    dt = dt.plusSeconds(59);
+                }
+                else {
+                    dt = dt.plusSeconds(bag.getSec());
+                }
             }
             sec = dt.getMillis() / 1000;
         }

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampParser.java

@@ -0,0 +1,232 @@
+package org.embulk.spi.time;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.joda.time.DateTimeZone;
+import org.junit.Test;
+
+/**
+ * TestTimestampParser tests org.embulk.spi.time.TimestampParser.
+ *
+ * Some test cases are imported from Ruby v2.3.1's test/date/test_date_strptime.rb. See its COPYING for license.
+ *
+ * @see <a href="https://svn.ruby-lang.org/cgi-bin/viewvc.cgi/tags/v2_3_1/test/date/test_date_strptime.rb?view=markup">test/date/test_date_strptime.rb</a>
+ * @see <a href="https://svn.ruby-lang.org/cgi-bin/viewvc.cgi/tags/v2_3_1/COPYING?view=markup">COPYING</a>
+ */
+public class TestTimestampParser {
+    @Test  // Imported from test__strptime__3 in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__3_iso8601() {
+        testToParse("2001-02-03", "%Y-%m-%d", 981158400L);
+        testToParse("2001-02-03T23:59:60", "%Y-%m-%dT%H:%M:%S", 981244799L);
+        testToParse("2001-02-03T23:59:60+09:00", "%Y-%m-%dT%H:%M:%S%Z", 981212399L);
+        testToParse("-2001-02-03T23:59:60+09:00", "%Y-%m-%dT%H:%M:%S%Z", -125309754001L);
+        testToParse("+012345-02-03T23:59:60+09:00", "%Y-%m-%dT%H:%M:%S%Z", 327406287599L);
+        testToParse("-012345-02-03T23:59:60+09:00", "%Y-%m-%dT%H:%M:%S%Z", -451734829201L);
+    }
+
+    @Test  // Imported from test__strptime__3 in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__3_ctime3_asctime3() {
+        testToParse("Thu Jul 29 14:47:19 1999", "%c", 933259639L);
+        testToParse("Thu Jul 29 14:47:19 -1999", "%c", -125231389961L);
+    }
+
+    @Test  // Imported from test__strptime__3 in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__3_date1() {
+        testToParse("Thu Jul 29 16:39:41 EST 1999", "%a %b %d %H:%M:%S %Z %Y", 933284381L);
+        testToParse("Thu Jul 29 16:39:41 MET DST 1999", "%a %b %d %H:%M:%S %Z %Y", 933259181L);
+        // Their time zones are "AMT" actually in Ruby v2.3.1's tests, but "AST" is used here instead.
+        // "AMT" is not recognized even by Ruby v2.3.1's zonetab.
+        testToParse("Thu Jul 29 16:39:41 AST 1999", "%a %b %d %H:%M:%S %Z %Y", 933280781L);
+        testToParse("Thu Jul 29 16:39:41 AST -1999", "%a %b %d %H:%M:%S %Z %Y", -125231368819L);
+        testToParse("Thu Jul 29 16:39:41 GMT+09 1999", "%a %b %d %H:%M:%S %Z %Y", 933233981L);
+        testToParse("Thu Jul 29 16:39:41 GMT+0908 1999", "%a %b %d %H:%M:%S %Z %Y", 933233501L);
+        testToParse("Thu Jul 29 16:39:41 GMT+090807 1999", "%a %b %d %H:%M:%S %Z %Y", 933233494L);
+        testToParse("Thu Jul 29 16:39:41 GMT-09 1999", "%a %b %d %H:%M:%S %Z %Y", 933298781L);
+        testToParse("Thu Jul 29 16:39:41 GMT-09:08 1999", "%a %b %d %H:%M:%S %Z %Y", 933299261L);
+        testToParse("Thu Jul 29 16:39:41 GMT-09:08:07 1999", "%a %b %d %H:%M:%S %Z %Y", 933299268L);
+        testToParse("Thu Jul 29 16:39:41 GMT-3.5 1999", "%a %b %d %H:%M:%S %Z %Y", 933278981L);
+        testToParse("Thu Jul 29 16:39:41 GMT-3,5 1999", "%a %b %d %H:%M:%S %Z %Y", 933278981L);
+        testToParse("Thu Jul 29 16:39:41 Mountain Daylight Time 1999", "%a %b %d %H:%M:%S %Z %Y", 933287981L);
+        testToParse("Thu Jul 29 16:39:41 E. Australia Standard Time 1999", "%a %b %d %H:%M:%S %Z %Y", 933230381L);
+    }
+
+    @Test  // Imported from test__strptime__3 in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__3_rfc822() {
+        testToParse("Thu, 29 Jul 1999 09:54:21 UT", "%a, %d %b %Y %H:%M:%S %Z", 933242061L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 GMT", "%a, %d %b %Y %H:%M:%S %Z", 933242061L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 PDT", "%a, %d %b %Y %H:%M:%S %Z", 933267261L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 z", "%a, %d %b %Y %H:%M:%S %Z", 933242061L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 +0900", "%a, %d %b %Y %H:%M:%S %Z", 933209661L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 +0430", "%a, %d %b %Y %H:%M:%S %Z", 933225861L);
+        testToParse("Thu, 29 Jul 1999 09:54:21 -0430", "%a, %d %b %Y %H:%M:%S %Z", 933258261L);
+        testToParse("Thu, 29 Jul -1999 09:54:21 -0430", "%a, %d %b %Y %H:%M:%S %Z", -125231391339L);
+    }
+
+    @Test  // Imported from test__strptime__3 in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__3_etc() {
+        testToParse("06-DEC-99", "%d-%b-%y", 944438400L);
+        testToParse("sUnDay oCtoBer 31 01", "%A %B %d %y", 1004486400L);
+        // Their "\u000b" are actually "\v" in Ruby v2.3.1's tests. "\v" is not recognized as a character in Java.
+        testToParse("October\t\n\u000b\f\r 15,\t\n\u000b\f\r99", "%B %d, %y", 939945600L);
+        testToParse("October\t\n\u000b\f\r 15,\t\n\u000b\f\r99", "%B%t%d,%n%y", 939945600L);
+
+        testToParse("09:02:11 AM", "%I:%M:%S %p", 81955357331L);
+        testToParse("09:02:11 A.M.", "%I:%M:%S %p", 81955357331L);
+        testToParse("09:02:11 PM", "%I:%M:%S %p", 81955400531L);
+        testToParse("09:02:11 P.M.", "%I:%M:%S %p", 81955400531L);
+
+        testToParse("12:33:44 AM", "%r", 81955326824L);
+        testToParse("01:33:44 AM", "%r", 81955330424L);
+        testToParse("11:33:44 AM", "%r", 81955366424L);
+        testToParse("12:33:44 PM", "%r", 81955370024L);
+        testToParse("01:33:44 PM", "%r", 81955373624L);
+        testToParse("11:33:44 PM", "%r", 81955409624L);
+
+        // Their time zones are "AMT" actually in Ruby v2.3.1's tests, but "AST" is used here instead.
+        // "AMT" is not recognized even by Ruby v2.3.1's zonetab.
+        testToParse("11:33:44 PM AST", "%I:%M:%S %p %Z", 81955424024L);
+        testToParse("11:33:44 P.M. AST", "%I:%M:%S %p %Z", 81955424024L);
+
+        testToParse("fri1feb034pm+5", "%a%d%b%y%H%p%Z", 1044097200L);
+    }
+
+    @Test  // Imported from test__strptime__width in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__width() {
+        testToParse("99", "%y", 917049600L);
+        testToParse("01", "%y", 980208000L);
+        testToParse("19 99", "%C %y", 917049600L);
+        testToParse("20 01", "%C %y", 980208000L);
+        testToParse("30 99", "%C %y", 35629718400L);
+        testToParse("30 01", "%C %y", 32537116800L);
+        testToParse("1999", "%C%y", 917049600L);
+        testToParse("2001", "%C%y", 980208000L);
+        testToParse("3099", "%C%y", 35629718400L);
+        testToParse("3001", "%C%y", 32537116800L);
+
+        testToParse("20060806", "%Y", 632995726752000L);
+        // Its " " is actually "\s" in Ruby v2.3.1's tests. "\s" is not recognized as a character in Java.
+        testToParse("20060806", "%Y ", 632995726752000L);
+        testToParse("20060806", "%Y%m%d", 1154822400L);
+        testToParse("2006908906", "%Y9%m9%d", 1154822400L);
+        testToParse("12006 08 06", "%Y %m %d", 316724342400L);
+        testToParse("12006-08-06", "%Y-%m-%d", 316724342400L);
+        testToParse("200608 6", "%Y%m%e", 1154822400L);
+
+        testToParse("2006333", "%Y%j", 1164758400L);
+        testToParse("20069333", "%Y9%j", 1164758400L);
+        testToParse("12006 333", "%Y %j", 316734278400L);
+        testToParse("12006-333", "%Y-%j", 316734278400L);
+
+        testToParse("232425", "%H%M%S", 81955409065L);
+        testToParse("23924925", "%H9%M9%S", 81955409065L);
+        testToParse("23 24 25", "%H %M %S", 81955409065L);
+        testToParse("23:24:25", "%H:%M:%S", 81955409065L);
+        testToParse(" 32425", "%k%M%S", 81955337065L);
+        testToParse(" 32425", "%l%M%S", 81955337065L);
+
+        // They are intentionally skipped as a month and a day of week are not sufficient to build a timestamp.
+        // [['FriAug', '%a%b'], [nil,8,nil,nil,nil,nil,nil,nil,5], __LINE__],
+        // [['FriAug', '%A%B'], [nil,8,nil,nil,nil,nil,nil,nil,5], __LINE__],
+        // [['FridayAugust', '%A%B'], [nil,8,nil,nil,nil,nil,nil,nil,5], __LINE__],
+        // [['FridayAugust', '%a%b'], [nil,8,nil,nil,nil,nil,nil,nil,5], __LINE__],
+    }
+
+    @Test  // Imported from test__strptime__fail in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test__strptime__fail() {
+        testToParse("2001.", "%Y.", 980208000L);
+        // Its " " is actually "\s" in Ruby v2.3.1's tests. "\s" is not recognized as a character in Java.
+        testToParse("2001. ", "%Y.", 980208000L);
+        testToParse("2001.", "%Y. ", 980208000L);
+        // Its " " is actually "\s" in Ruby v2.3.1's tests. "\s" is not recognized as a character in Java.
+        testToParse("2001. ", "%Y. ", 980208000L);
+
+        failToParse("2001", "%Y.");
+        // Its " " is actually "\s" in Ruby v2.3.1's tests. "\s" is not recognized as a character in Java.
+        failToParse("2001 ", "%Y.");
+        failToParse("2001", "%Y. ");
+        // Its " " is actually "\s" in Ruby v2.3.1's tests. "\s" is not recognized as a character in Java.
+        failToParse("2001 ", "%Y. ");
+
+        failToParse("2001-13-31", "%Y-%m-%d");
+        failToParse("2001-12-00", "%Y-%m-%d");
+        failToParse("2001-12-32", "%Y-%m-%d");
+        failToParse("2001-12-00", "%Y-%m-%e");
+        failToParse("2001-12-32", "%Y-%m-%e");
+        failToParse("2001-12-31", "%y-%m-%d");
+
+        failToParse("2004-000", "%Y-%j");
+        failToParse("2004-367", "%Y-%j");
+        failToParse("2004-366", "%y-%j");
+
+        testToParse("24:59:59", "%H:%M:%S", 81955414799L);
+        testToParse("24:59:59", "%k:%M:%S", 81955414799L);
+        testToParse("24:59:60", "%H:%M:%S", 81955414799L);
+        testToParse("24:59:60", "%k:%M:%S", 81955414799L);
+
+        failToParse("24:60:59", "%H:%M:%S");
+        failToParse("24:60:59", "%k:%M:%S");
+        failToParse("24:59:61", "%H:%M:%S");
+        failToParse("24:59:61", "%k:%M:%S");
+        failToParse("00:59:59", "%I:%M:%S");
+        failToParse("13:59:59", "%I:%M:%S");
+        failToParse("00:59:59", "%l:%M:%S");
+        failToParse("13:59:59", "%l:%M:%S");
+
+        testToParse("0", "%U", 81955324800L);
+        failToParse("54", "%U");
+        testToParse("0", "%W", 81955324800L);
+        failToParse("54", "%W");
+        failToParse("0", "%V");
+        failToParse("54", "%V");
+        failToParse("0", "%u");
+        testToParse("7", "%u", 81955324800L);
+        testToParse("0", "%w", 81955324800L);
+        failToParse("7", "%w");
+
+        failToParse("Sanday", "%A");
+        failToParse("Jenuary", "%B");
+        testToParse("Sundai", "%A", 81955324800L);
+        testToParse("Januari", "%B", 81955324800L);
+        failToParse("Sundai,", "%A,");
+        failToParse("Januari,", "%B,");
+    }
+
+    @Test  // Imported partially from test_strptime in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test_strptime() {
+        testToParse("2002-03-14T11:22:33Z", "%Y-%m-%dT%H:%M:%S%Z", 1016104953L);
+        testToParse("2002-03-14T11:22:33+09:00", "%Y-%m-%dT%H:%M:%S%Z", 1016072553L);
+        testToParse("2002-03-14T11:22:33-09:00", "%FT%T%Z", 1016137353L);
+        testToParse("2002-03-14T11:22:33.123456789-09:00", "%FT%T.%N%Z", 1016137353L, 123456789);
+    }
+
+    @Test  // Imported from test_strptime__minus in Ruby v2.3.1's test/date/test_date_strptime.rb.
+    public void test_strptime__minus() {
+        testToParse("-1", "%s", -1L);
+        testToParse("-86400", "%s", -86400L);
+
+        testToParse("-999", "%Q", 0L, -999000000);
+        testToParse("-1000", "%Q", -1L);
+    }
+
+    private void testToParse(final String string, final String format, final long second, final int nanoOfSecond) {
+        final TimestampParser parser = new TimestampParser(format, DateTimeZone.UTC, "4567-01-23");
+        final Timestamp timestamp = parser.parse(string);
+        assertEquals(second, timestamp.getEpochSecond());
+        assertEquals(nanoOfSecond,timestamp.getNano());
+    }
+
+    private void testToParse(final String string, final String format, final long second) {
+        testToParse(string, format, second, 0);
+    }
+
+    private void failToParse(final String string, final String format) {
+        final TimestampParser parser = new TimestampParser(format, DateTimeZone.UTC, "4567-01-23");
+        try {
+            parser.parse(string);
+        } catch (TimestampParseException ex) {
+            return;
+        }
+        fail();
+    }
+}

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.37
     release/release-0.8.36
     release/release-0.8.35
     release/release-0.8.34

data/embulk-docs/src/release/release-0.8.37.rst

@@ -0,0 +1,20 @@
+Release 0.8.37
+==================================
+
+General Changes
+----------------
+
+* Parse Ruby-style time zone names in addition [#833] [#840]
+* Remove dead code around the Ruby timestamp parser replaced [#813] [#823]
+
+Bug Fixes
+----------
+
+* Fix parsing leap seconds to get the same Timestamp with pure Ruby's [#842]
+* Process yday (day of the year) in TimestampParser as well [#834] [#843]
+* Fix nanoseconds set by StrptimeParser's %Q [#848]
+
+
+Release Date
+------------------
+2017-11-21

data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeFormat.java

@@ -36,7 +36,7 @@ enum StrptimeFormat
     FORMAT_NANOSEC, // %N
     FORMAT_MERIDIAN_LOWER_CASE, // %P
     FORMAT_MERIDIAN, // %p
-    FORMAT_MICROSEC_EPOCH, // %Q Only for Date/DateTime from here
+    FORMAT_MILLISEC_EPOCH, // %Q Only for Date/DateTime from here
     FORMAT_SECONDS, // %S
     FORMAT_EPOCH, // %s
     FORMAT_WEEK_YEAR_S, // %U

data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeParser.java

@@ -576,7 +576,7 @@ public class StrptimeParser
                         }
                         break;
                     }
-                    case FORMAT_MICROSEC_EPOCH: { // %Q - Number of microseconds since 1970-01-01 00:00:00 UTC.
+                    case FORMAT_MILLISEC_EPOCH: { // %Q - Number of milliseconds since 1970-01-01 00:00:00 UTC.
                         boolean negative = false;
                         if (isMinus(text, pos)) {
                             negative = true;

data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeToken.java

@@ -39,7 +39,7 @@ public class StrptimeToken
         CONVERSION2TOKEN['N'] = new StrptimeToken(FORMAT_NANOSEC);
         CONVERSION2TOKEN['P'] = new StrptimeToken(FORMAT_MERIDIAN_LOWER_CASE);
         CONVERSION2TOKEN['p'] = new StrptimeToken(FORMAT_MERIDIAN);
-        CONVERSION2TOKEN['Q'] = new StrptimeToken(FORMAT_MICROSEC_EPOCH);
+        CONVERSION2TOKEN['Q'] = new StrptimeToken(FORMAT_MILLISEC_EPOCH);
         CONVERSION2TOKEN['S'] = new StrptimeToken(FORMAT_SECONDS);
         CONVERSION2TOKEN['s'] = new StrptimeToken(FORMAT_EPOCH);
         CONVERSION2TOKEN['U'] = new StrptimeToken(FORMAT_WEEK_YEAR_S);

data/lib/embulk/java/bootstrap.rb

@@ -1,7 +1,7 @@
 module Embulk
   module Java
     require 'embulk/java/imports'
-    require 'embulk/java/time_helper'
+    require 'time'
 
     module Injected
       # Following constats are set by org.embulk.jruby.JRubyScriptingModule:

data/lib/embulk/version.rb

@@ -3,7 +3,7 @@
 module Embulk
   @@warned = false
 
-  VERSION_INTERNAL = '0.8.36'
+  VERSION_INTERNAL = '0.8.37'
 
   DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
   def self.const_missing(name)

data/test/{run-test.rb → monkey_strptime/run-test.rb}

@@ -1,10 +1,11 @@
-base_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", ".."))
 lib_dir = File.join(base_dir, "lib")
 test_dir = File.join(base_dir, "test")
 
 $LOAD_PATH.unshift(lib_dir)
 $LOAD_PATH.unshift(test_dir)
 
+require "helper"
 require "date"
 require "test/unit"
 
@@ -21,7 +22,7 @@ module DateExt
 end
 Date.send(:include, DateExt)
 
-Dir.glob("#{base_dir}/test/**/test{_,-}*.rb") do |file|
+Dir.glob("#{base_dir}/test/monkey_strptime/**/test{_,-}*.rb") do |file|
   require file.sub(/\.rb$/,"")
 end
 

data/test/vanilla/run-test.rb

@@ -0,0 +1,14 @@
+base_dir = File.expand_path(File.join(File.dirname(__FILE__), "..", ".."))
+lib_dir = File.join(base_dir, "lib")
+test_dir = File.join(base_dir, "test")
+
+$LOAD_PATH.unshift(lib_dir)
+$LOAD_PATH.unshift(test_dir)
+
+require "test/unit"
+
+Dir.glob("#{base_dir}/test/vanilla/**/test{_,-}*.rb") do |file|
+  require file.sub(/\.rb$/,"")
+end
+
+exit Test::Unit::AutoRunner.run

data/test/vanilla/time/test_timestamp_parser.rb

@@ -0,0 +1,102 @@
+require 'helper'
+require 'date'
+require 'time'
+
+# TimestampParserTest test org.embulk.spi.time.TimestampParser by comparing with Ruby's DateTime.strptime.
+#
+# Some test cases are imported from Ruby v2.3.1's test/date/test_date_strptime.rb. See its COPYING for license.
+#
+# *{test/date/test_date_strptime.rb}[https://svn.ruby-lang.org/cgi-bin/viewvc.cgi/tags/v2_3_1/test/date/test_date_strptime.rb?view=markup]
+# *{COPYING}[https://svn.ruby-lang.org/cgi-bin/viewvc.cgi/tags/v2_3_1/COPYING?view=markup]
+
+class TimestampParserTest < ::Test::Unit::TestCase
+  def test__strptime__3
+    [
+      # iso8601
+      ['2001-02-03', '%Y-%m-%d'],
+      ['2001-02-03T23:59:60', '%Y-%m-%dT%H:%M:%S'],
+      ['2001-02-03T23:59:60+09:00', '%Y-%m-%dT%H:%M:%S%Z'],
+      ['-2001-02-03T23:59:60+09:00', '%Y-%m-%dT%H:%M:%S%Z'],
+      ['+012345-02-03T23:59:60+09:00', '%Y-%m-%dT%H:%M:%S%Z'],
+      ['-012345-02-03T23:59:60+09:00', '%Y-%m-%dT%H:%M:%S%Z'],
+
+      # ctime(3), asctime(3)
+      ['Thu Jul 29 14:47:19 1999', '%c'],
+      ['Thu Jul 29 14:47:19 -1999', '%c'],
+
+      # date(1)
+      ['Thu Jul 29 16:39:41 EST 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 MET DST 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      # Their time zones are "AMT" actually in Ruby v2.3.1's tests, but "AST" is used here instead.
+      # "AMT" is not recognized even by Ruby v2.3.1's zonetab.
+      ['Thu Jul 29 16:39:41 AST 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 AST -1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT+09 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT+0908 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT+090807 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT-09 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT-09:08 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT-09:08:07 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT-3.5 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 GMT-3,5 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 Mountain Daylight Time 1999', '%a %b %d %H:%M:%S %Z %Y'],
+      ['Thu Jul 29 16:39:41 E. Australia Standard Time 1999', '%a %b %d %H:%M:%S %Z %Y'],
+
+      # rfc822
+      ['Thu, 29 Jul 1999 09:54:21 UT', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 GMT', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 PDT', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 z', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 +0900', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 +0430', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul 1999 09:54:21 -0430', '%a, %d %b %Y %H:%M:%S %Z'],
+      ['Thu, 29 Jul -1999 09:54:21 -0430', '%a, %d %b %Y %H:%M:%S %Z'],
+
+      # etc
+      ['06-DEC-99', '%d-%b-%y'],
+      ['sUnDay oCtoBer 31 01', '%A %B %d %y'],
+      ["October\t\n\v\f\r 15,\t\n\v\f\r99", '%B %d, %y'],
+      ["October\t\n\v\f\r 15,\t\n\v\f\r99", '%B%t%d,%n%y'],
+
+      ['09:02:11 AM', '%I:%M:%S %p'],
+      ['09:02:11 A.M.', '%I:%M:%S %p'],
+      ['09:02:11 PM', '%I:%M:%S %p'],
+      ['09:02:11 P.M.', '%I:%M:%S %p'],
+
+      ['12:33:44 AM', '%r'],
+      ['01:33:44 AM', '%r'],
+      ['11:33:44 AM', '%r'],
+      ['12:33:44 PM', '%r'],
+      ['01:33:44 PM', '%r'],
+      ['11:33:44 PM', '%r'],
+
+      # Their time zones are "AMT" actually in Ruby v2.3.1's tests, but "AST" is used here instead.
+      # "AMT" is not recognized even by Ruby v2.3.1's zonetab.
+      ['11:33:44 PM AST', '%I:%M:%S %p %Z'],
+      ['11:33:44 P.M. AST', '%I:%M:%S %p %Z'],
+
+      ['fri1feb034pm+5', '%a%d%b%y%H%p%Z'],
+    ].each do |string, format|
+      before_date = Date.today
+      after_date = nil
+      while before_date != after_date do
+        before_date = Date.today
+        expected_datetime = DateTime.strptime(string, format)
+        expected_time = expected_datetime.new_offset(0).to_time.utc
+        after_date = Date.today
+        if expected_datetime.to_date == after_date  # DateTime.strptime fills today's date if date is not contained.
+          # Getting the Time of 00:00:00 UTC on after_date.
+          expected_time = expected_time - after_date.to_datetime.new_offset(0).to_time.utc
+        end
+        expected_epoch = expected_time.to_i
+      end
+
+      timestamp_parser =
+        Java::org.embulk.spi.time.TimestampParser.new(format, Java::org.joda.time.DateTimeZone::UTC, "1970-01-01")
+      actual_timestamp = timestamp_parser.parse(string)
+      actual_epoch = actual_timestamp.getEpochSecond()
+
+      assert_equal(expected_epoch, actual_epoch, string)
+    end
+  end
+end

metadata

@@ -1,127 +1,127 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.36
+  version: 0.8.37
 platform: java
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-10-24 00:00:00.000000000 Z
+date: 2017-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
+  name: bundler
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.6
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.0
+  name: msgpack
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.1.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: liquid
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 4.0.0
+  name: liquid
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 4.0.0
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: rjack-icu
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 4.54.1.1
+  name: rjack-icu
+  prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 4.54.1.1
-  prerelease: false
-  type: :runtime
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
+  name: rake
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.9
+  name: test-unit
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.0.9
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
+  name: yard
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.8.7
-  prerelease: false
-  type: :development
 - !ruby/object:Gem::Dependency
-  name: kramdown
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.0
+  name: kramdown
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 1.5.0
-  prerelease: false
-  type: :development
 description: Embulk is an open-source, plugin-based bulk data loader to scale and
   simplify data management across heterogeneous data stores. It can collect and ship
   any kinds of data in high throughput with transaction control.
@@ -157,10 +157,10 @@ files:
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang-2.4.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-cli-0.8.36.jar
-- classpath/embulk-core-0.8.36.jar
-- classpath/embulk-jruby-strptime-0.8.36.jar
-- classpath/embulk-standards-0.8.36.jar
+- classpath/embulk-cli-0.8.37.jar
+- classpath/embulk-core-0.8.37.jar
+- classpath/embulk-jruby-strptime-0.8.37.jar
+- classpath/embulk-standards-0.8.37.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -348,8 +348,6 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/json/JsonParser.java
 - embulk-core/src/main/java/org/embulk/spi/json/RubyValueApi.java
 - embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java
-- embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java
-- embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java
 - embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java
 - embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java
 - embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java
@@ -439,6 +437,7 @@ files:
 - embulk-core/src/test/java/org/embulk/spi/time/TestTimestamp.java
 - embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java
 - embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParserDeprecated.java
+- embulk-core/src/test/java/org/embulk/spi/time/TestTimestampParser.java
 - embulk-core/src/test/java/org/embulk/spi/type/TestTypeSerDe.java
 - embulk-core/src/test/java/org/embulk/spi/unit/TestByteSize.java
 - embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java
@@ -568,6 +567,7 @@ files:
 - embulk-docs/src/release/release-0.8.34.rst
 - embulk-docs/src/release/release-0.8.35.rst
 - embulk-docs/src/release/release-0.8.36.rst
+- embulk-docs/src/release/release-0.8.37.rst
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst
 - embulk-docs/src/release/release-0.8.6.rst
@@ -782,7 +782,6 @@ files:
 - lib/embulk/input_plugin.rb
 - lib/embulk/java/bootstrap.rb
 - lib/embulk/java/imports.rb
-- lib/embulk/java/time_helper.rb
 - lib/embulk/java_plugin.rb
 - lib/embulk/logger.rb
 - lib/embulk/output_plugin.rb
@@ -795,13 +794,15 @@ files:
 - lib/embulk/schema.rb
 - lib/embulk/version.rb
 - settings.gradle
-- test/guess/test_csv_all_strings.rb
-- test/guess/test_csv_guess.rb
-- test/guess/test_schema_guess.rb
-- test/guess/test_time_format_guess.rb
 - test/helper.rb
-- test/mri/date/test_date_strptime.rb
-- test/run-test.rb
+- test/monkey_strptime/mri/date/test_date_strptime.rb
+- test/monkey_strptime/run-test.rb
+- test/vanilla/guess/test_csv_all_strings.rb
+- test/vanilla/guess/test_csv_guess.rb
+- test/vanilla/guess/test_schema_guess.rb
+- test/vanilla/guess/test_time_format_guess.rb
+- test/vanilla/run-test.rb
+- test/vanilla/time/test_timestamp_parser.rb
 homepage: https://github.com/embulk/embulk
 licenses:
 - Apache 2.0
@@ -827,10 +828,12 @@ signing_key:
 specification_version: 4
 summary: Embulk, a plugin-based parallel bulk data loader
 test_files:
-- test/guess/test_csv_all_strings.rb
-- test/guess/test_csv_guess.rb
-- test/guess/test_schema_guess.rb
-- test/guess/test_time_format_guess.rb
 - test/helper.rb
-- test/mri/date/test_date_strptime.rb
-- test/run-test.rb
+- test/monkey_strptime/mri/date/test_date_strptime.rb
+- test/monkey_strptime/run-test.rb
+- test/vanilla/guess/test_csv_all_strings.rb
+- test/vanilla/guess/test_csv_guess.rb
+- test/vanilla/guess/test_schema_guess.rb
+- test/vanilla/guess/test_time_format_guess.rb
+- test/vanilla/run-test.rb
+- test/vanilla/time/test_timestamp_parser.rb

data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelper.java

@@ -1,8 +0,0 @@
-package org.embulk.spi.time;
-
-public interface JRubyTimeParserHelper
-{
-    long strptimeUsec(String text) throws TimestampParseException;
-
-    String getZone();
-}

data/embulk-core/src/main/java/org/embulk/spi/time/JRubyTimeParserHelperFactory.java

@@ -1,6 +0,0 @@
-package org.embulk.spi.time;
-
-public interface JRubyTimeParserHelperFactory
-{
-    JRubyTimeParserHelper newInstance(String formatString, int year, int mon, int day, int hour, int min, int sec, int usec);
-}

data/lib/embulk/java/time_helper.rb

@@ -1,79 +0,0 @@
-module Embulk
-  module Java
-    #require 'embulk/java/imports'
-    require 'time'  # Date._strptime
-
-    class TimeParserHelper
-      #include Java::JRubyTimeParserHelper
-      include org.embulk.spi.time.JRubyTimeParserHelper
-
-      class Factory
-        #include Java::JRubyTimeParserHelperFactory
-        include org.embulk.spi.time.JRubyTimeParserHelperFactory
-
-        def newInstance(format_string, year, mon, day, hour, min, sec, usec)
-          default_time = Time.utc(year, mon, day, hour, min, sec, usec)
-          TimeParserHelper.new(format_string, default_time)
-        end
-      end
-
-      def initialize(format_string, default_time)
-        @format_string = format_string
-        @default_time = default_time
-      end
-
-      # Override
-      def strptimeUsec(text)
-        hash = Date._strptime(text, @format_string)
-        unless hash
-          raise Java::TimestampParseException.new("Failed to parse '" + text + "'")
-        end
-
-        if seconds = hash[:seconds] # if %s, it's Integer. if %Q, Rational.
-          sec_fraction = hash[:sec_fraction]  # Rational
-          usec = sec_fraction * 1_000_000 if sec_fraction
-          return (seconds * 1_000_000).to_i + usec.to_i
-
-        else
-          year = hash[:year]
-          mon = hash[:mon]
-          day = hash[:mday]
-          hour = hash[:hour]
-          min = hash[:min]
-          sec = hash[:sec]
-          sec_fraction = hash[:sec_fraction]
-          usec = sec_fraction * 1_000_000 if sec_fraction
-          zone = hash[:zone]
-
-          now = @default_time
-          begin
-            break if year; year = now.year
-            break if mon; mon = now.mon
-            break if day; day = now.day
-            break if hour; hour = now.hour
-            break if min; min = now.min
-            break if sec; sec = now.sec
-            break if sec_fraction; usec = now.tv_usec
-          end until true
-
-          year ||= 1970
-          mon ||= 1
-          day ||= 1
-          hour ||= 0
-          min ||= 0
-          sec ||= 0
-          usec ||= 0
-
-          @zone = zone
-          time = Time.utc(year, mon, day, hour, min, sec, usec)
-          return time.tv_sec * 1_000_000 + time.tv_usec
-        end
-      end
-
-      # Override
-      def getZone
-        @zone
-      end
-    end
-  end
-end