embulk 0.8.26-java → 0.8.27-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67e9c5003f422b52f56fd7347f77dda2ea07d86e
-  data.tar.gz: ac13f8e09bcc9998ba716fc96e6e98e0546605ef
+  metadata.gz: 61453ab9514646a642d390b971aeca9cb45ea066
+  data.tar.gz: 6c034da535e1687bcf3f6f9a75f12557b73d1f17
 SHA512:
-  metadata.gz: 6a275c7189877da26bed04a116dab321b0bf5d9c8f378143aa743c2fa05ae0806fb8d27eabaad76c6bdd7b06971a1b8f5ea2aa8a52152aa7e67a129545e417bc
-  data.tar.gz: 6ce56d2cfec57fc2db846f7be7aaf5209e30ac5180b28f0d9930f7cebd333f614e443dcb1e57b49bc58683e4b1a595239809bbe50d4915f9bd21e5d59597abb7
+  metadata.gz: 10c5d683990626cbe25dcfc6dcdc5f967d74d7420743f6c33ba1a1997765156b5e4ee1a0914737baa4f1ba80206864e404941b309e1b1efb9c05e2efae293c87
+  data.tar.gz: 9f21b019c2752622b0108b649859fbb1a39ea2420c2381e71df50ec683ee7f9a742fe0038096a5e6cf0e294fe5068ddf6f32ca8ee9e71bfd6f83cc01167b5027

data/build.gradle

@@ -11,12 +11,12 @@ apply plugin: "com.github.jruby-gradle.jar"
 apply plugin: 'com.jfrog.bintray'
 apply plugin: 'com.github.johnrengelman.shadow'
 
-def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli"), project(":embulk-test")]
-def release_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-test")]
+def java_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-cli"), project(":embulk-test"), project(":embulk-jruby-strptime")]
+def release_projects = [project(":embulk-core"), project(":embulk-standards"), project(":embulk-test"), project(":embulk-jruby-strptime")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.26'
+    version = '0.8.27'
 
     ext {
         jrubyVersion = '9.1.5.0'
@@ -28,6 +28,7 @@ allprojects {
 
     test {
         maxHeapSize = "1536m"
+        forkEvery = 1 // test processes are forked by each test class (default is 0)
     }
 
     //

data/embulk-core/build.gradle

@@ -16,6 +16,8 @@ import com.github.jrubygradle.JRubyPrepare
 
 // determine which dependencies have updates: $ gradle dependencyUpdates
 dependencies {
+    compile project(':embulk-jruby-strptime')
+
     compile 'org.embulk:guice-bootstrap:0.1.1'
     compile 'com.google.guava:guava:18.0'
     compile 'com.google.inject:guice:4.0'

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -1,18 +1,24 @@
 package org.embulk.spi.time;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
 import java.util.Date;
+import java.util.List;
 import java.util.Locale;
 import java.util.TimeZone;
-import java.util.Calendar;
-import java.text.SimpleDateFormat;
-import java.text.ParseException;
-import org.joda.time.DateTimeZone;
-import com.google.common.base.Optional;
-import org.jruby.embed.ScriptingContainer;
 import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigInject;
+import org.embulk.spi.time.StrptimeParser.FormatBag;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
 
 public class TimestampParser
@@ -62,8 +68,11 @@ public class TimestampParser
         public Optional<String> getDate();
     }
 
-    private final JRubyTimeParserHelper helper;
     private final DateTimeZone defaultTimeZone;
+    private final String format;
+    private final StrptimeParser parser;
+    private final Calendar calendar;
+    private final List<StrptimeToken> compiledPattern;
 
     @Deprecated
     public TimestampParser(String format, ParserTask task)
@@ -71,9 +80,11 @@ public class TimestampParser
         this(task.getJRuby(), format, task.getDefaultTimeZone());
     }
 
-    TimestampParser(Task task)
+    @VisibleForTesting
+    static TimestampParser createTimestampParserForTesting(Task task)
+
     {
-        this(task.getJRuby(), task.getDefaultTimestampFormat(), task.getDefaultTimeZone(), task.getDefaultDate());
+        return new TimestampParser(task.getJRuby(), task.getDefaultTimestampFormat(), task.getDefaultTimeZone(), task.getDefaultDate());
     }
 
     public TimestampParser(Task task, TimestampColumnOption columnOption)
@@ -91,7 +102,11 @@ public class TimestampParser
 
     public TimestampParser(ScriptingContainer jruby, String format, DateTimeZone defaultTimeZone, String defaultDate)
     {
-        JRubyTimeParserHelperFactory helperFactory = (JRubyTimeParserHelperFactory) jruby.runScriptlet("Embulk::Java::TimeParserHelper::Factory.new");
+        // TODO get default current time from ExecTask.getExecTimestamp
+        this.format = format;
+        this.parser = new StrptimeParser();
+        this.compiledPattern = this.parser.compilePattern(format);
+        this.defaultTimeZone = defaultTimeZone;
 
         // calculate default date
         SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH);
@@ -103,15 +118,8 @@ public class TimestampParser
         catch (ParseException ex) {
             throw new ConfigException("Invalid date format. Expected yyyy-MM-dd: " + defaultDate);
         }
-        Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ENGLISH);
-        cal.setTime(utc);
-        int year = cal.get(Calendar.YEAR);
-        int month = cal.get(Calendar.MONTH) + 1;
-        int day = cal.get(Calendar.DAY_OF_MONTH);
-
-        // TODO get default current time from ExecTask.getExecTimestamp
-        this.helper = (JRubyTimeParserHelper) helperFactory.newInstance(format, year, month, day, 0, 0, 0, 0);  // TODO default time zone
-        this.defaultTimeZone = defaultTimeZone;
+        this.calendar = Calendar.getInstance(TimeZone.getTimeZone("UTC"), Locale.ENGLISH);
+        this.calendar.setTime(utc);
     }
 
     public DateTimeZone getDefaultTimeZone()
@@ -121,10 +129,21 @@ public class TimestampParser
 
     public Timestamp parse(String text) throws TimestampParseException
     {
-        long localUsec = helper.strptimeUsec(text);
-        String zone = helper.getZone();
+        if (isNullOrEmpty(text)) {
+            throw new TimestampParseException("text is null or empty string.");
+        }
+
+        final FormatBag bag = parser.parse(compiledPattern, text);
+        if (bag == null) {
+            throw new TimestampParseException("Cannot parse '" + text + "' by '" + format + "'");
+        }
+        bag.setYearIfNotSet(calendar.get(Calendar.YEAR));
+        bag.setMonthIfNotSet(calendar.get(Calendar.MONTH) + 1);
+        bag.setMdayIfNotSet(calendar.get(Calendar.DAY_OF_MONTH));
 
-        DateTimeZone timeZone = defaultTimeZone;
+        final LocalTime local = createLocalTimeFromFormatBag(bag);
+        final String zone = local.getZone();
+        final DateTimeZone timeZone;
         if (zone != null) {
             // TODO cache parsed zone?
             timeZone = parseDateTimeZone(zone);
@@ -132,11 +151,91 @@ public class TimestampParser
                 throw new TimestampParseException("Invalid time zone name '" + text + "'");
             }
         }
+        else {
+            timeZone = defaultTimeZone;
+        }
 
-        long localSec = localUsec / 1000000;
-        long usec = localUsec % 1000000;
-        long sec = timeZone.convertLocalToUTC(localSec*1000, false) / 1000;
+        final long sec = timeZone.convertLocalToUTC(local.getSeconds() * 1000, false) / 1000;
+        return Timestamp.ofEpochSecond(sec, local.getNsecFraction());
+    }
 
-        return Timestamp.ofEpochSecond(sec, usec * 1000);
+    public LocalTime createLocalTimeFromFormatBag(FormatBag bag)
+    {
+        final long secFractionNsec;
+        if (FormatBag.has(bag.getSecFraction())) {
+            secFractionNsec = bag.getSecFraction() * (int)Math.pow(10, 9 - bag.getSecFractionSize());
+        }
+        else {
+            secFractionNsec = 0;
+        }
+
+        final long sec;
+        if (bag.hasSeconds()) {
+            if (FormatBag.has(bag.getSecondsSize())) {
+                sec = bag.getSeconds() / (int)Math.pow(10, bag.getSecondsSize());
+            }
+            else { // int
+                sec = bag.getSeconds();
+            }
+
+        } else {
+            final int year;
+            if (FormatBag.has(bag.getYear())) {
+                year = bag.getYear();
+            }
+            else {
+                year = 1970;
+            }
+
+            // set up with min this and then add to allow rolling over
+            DateTime dt = new DateTime(year, 1, 1, 0, 0, 0, 0, DateTimeZone.UTC);
+            if (FormatBag.has(bag.getMon())) {
+                dt = dt.plusMonths(bag.getMon() - 1);
+            }
+            if (FormatBag.has(bag.getMDay())) {
+                dt = dt.plusDays(bag.getMDay() - 1);
+            }
+            if (FormatBag.has(bag.getHour())) {
+                dt = dt.plusHours(bag.getHour());
+            }
+            if (FormatBag.has(bag.getMin())) {
+                dt = dt.plusMinutes(bag.getMin());
+            }
+            if (FormatBag.has(bag.getSec())) {
+                dt = dt.plusSeconds(bag.getSec());
+            }
+            sec = dt.getMillis() / 1000;
+        }
+
+        return new LocalTime(sec, secFractionNsec, bag.getZone());
+    }
+
+    private static class LocalTime
+    {
+        private final long seconds;
+        private final long nsecFraction;
+        private final String zone;  // +0900, JST, UTC
+
+        public LocalTime(long seconds, long nsecFraction, String zone)
+        {
+            this.seconds = seconds;
+            this.nsecFraction = nsecFraction;
+            this.zone = zone;
+        }
+
+        public long getSeconds()
+        {
+            return seconds;
+        }
+
+        public long getNsecFraction()
+        {
+            return nsecFraction;
+        }
+
+        public String getZone()
+        {
+            return zone;
+        }
     }
 }

data/embulk-core/src/test/java/org/embulk/spi/time/TestTimestampFormatterParser.java

@@ -42,7 +42,7 @@ public class TestTimestampFormatterParser
             .set("default_timestamp_format", "%Y-%m-%d %H:%M:%S %z");  // %Z is OS-dependent
         ParserTestTask task = config.loadConfig(ParserTestTask.class);
 
-        TimestampParser parser = new TimestampParser(task);
+        TimestampParser parser = TimestampParser.createTimestampParserForTesting(task);
         assertEquals(Timestamp.ofEpochSecond(1416365189, 0), parser.parse("2014-11-19 02:46:29 +0000"));
     }
 
@@ -57,7 +57,7 @@ public class TestTimestampFormatterParser
         assertEquals("1416365189", formatter.format(Timestamp.ofEpochSecond(1416365189)));
 
         ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
-        TimestampParser parser = new TimestampParser(ptask);
+        TimestampParser parser = TimestampParser.createTimestampParserForTesting(ptask);
         assertEquals(Timestamp.ofEpochSecond(1416365189), parser.parse("1416365189"));
     }
 
@@ -69,7 +69,7 @@ public class TestTimestampFormatterParser
             .set("default_date", "2016-02-03");
 
         ParserTestTask ptask = config.loadConfig(ParserTestTask.class);
-        TimestampParser parser = new TimestampParser(ptask);
+        TimestampParser parser = TimestampParser.createTimestampParserForTesting(ptask);
         assertEquals(Timestamp.ofEpochSecond(1454467589, 0), parser.parse("02:46:29 +0000"));
     }
 }

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.27
     release/release-0.8.26
     release/release-0.8.25
     release/release-0.8.24

data/embulk-docs/src/release/release-0.8.27.rst

@@ -0,0 +1,15 @@
+Release 0.8.27
+==================================
+
+General Changes
+------------------
+
+* Add Java timestamp parser [#611]
+* Add the test for the timestamp parser [#712]
+* Fix build.gradle to set forkEvery=1 to improve CI build stability [#714]
+* Fix Javadoc in embulk-jruby-strptime classes to avoid Javadoc errors [#719]
+
+
+Release Date
+------------------
+2017-07-09

data/embulk-jruby-strptime/build.gradle

@@ -0,0 +1,3 @@
+dependencies {
+    compile 'org.jruby:jruby-complete:' + project.jrubyVersion
+}

data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/RubyDateParser.java

@@ -0,0 +1,121 @@
+package org.embulk.spi.time;
+
+import org.jruby.runtime.ThreadContext;
+
+import java.util.HashMap;
+import java.util.List;
+
+import static org.embulk.spi.time.StrptimeParser.FormatBag.has;
+import static org.jruby.RubyRational.newRationalCanonicalize;
+
+/**
+ * This class has {@code StrptimeParser} and provides methods that are calls from JRuby.
+ *
+ * TODO
+ * This class is tentatively required for executing test/mri/date/test_date_strptime.rb
+ * (#617). The {@code StrptimeParser} and {@code RubyDateParser} will be merged into
+ * JRuby (jruby/jruby#4591). embulk-jruby-strptime is removed when Embulk start using
+ * the JRuby that bundles embulk-jruby-strptime.
+ */
+public class RubyDateParser
+{
+    private final StrptimeParser strptimeParser;
+
+    public RubyDateParser()
+    {
+        this.strptimeParser = new StrptimeParser();
+    }
+
+    /**
+     * Date._strptime method in JRuby 9.1.5.0's lib/ruby/stdlib/date/format.rb is replaced
+     * with this method. This is Java implementation of date__strptime method in MRI 2.3.1's
+     * ext/date/date_strptime.c.
+     * @see <a href="https://github.com/jruby/jruby/blob/036ce39f0476d4bd718e23e64caff36bb50b8dbc/lib/ruby/stdlib/date/format.rb">format.rb</a>
+     * @see <a href="https://github.com/ruby/ruby/blob/394fa89c67722d35bdda89f10c7de5c304a5efb1/ext/date/date_strptime.c">date_strptime.c</a>
+     */
+    public HashMap<String, Object> parse(ThreadContext context, final String format, final String text)
+    {
+        final List<StrptimeToken> compiledPattern = strptimeParser.compilePattern(format);
+        final StrptimeParser.FormatBag bag = strptimeParser.parse(compiledPattern, text);
+        if (bag != null) {
+            return convertFormatBagToHash(context, bag);
+        }
+        else {
+            return null;
+        }
+    }
+
+    private HashMap<String, Object> convertFormatBagToHash(ThreadContext context, StrptimeParser.FormatBag bag)
+    {
+        final HashMap<String, Object> map = new HashMap<>();
+
+        if (has(bag.getMDay())) {
+            map.put("mday", bag.getMDay());
+        }
+        if (has(bag.getWDay())) {
+            map.put("wday", bag.getWDay());
+        }
+        if (has(bag.getCWDay())) {
+            map.put("cwday", bag.getCWDay());
+        }
+        if (has(bag.getYDay())) {
+            map.put("yday", bag.getYDay());
+        }
+        if (has(bag.getCWeek())) {
+            map.put("cweek", bag.getCWeek());
+        }
+        if (has(bag.getCWYear())) {
+            map.put("cwyear", bag.getCWYear());
+        }
+        if (has(bag.getMin())) {
+            map.put("min", bag.getMin());
+        }
+        if (has(bag.getMon())) {
+            map.put("mon", bag.getMon());
+        }
+        if (has(bag.getHour())) {
+            map.put("hour", bag.getHour());
+        }
+        if (has(bag.getYear())) {
+            map.put("year", bag.getYear());
+        }
+        if (has(bag.getSec())) {
+            map.put("sec", bag.getSec());
+        }
+        if (has(bag.getWNum0())) {
+            map.put("wnum0", bag.getWNum0());
+        }
+        if (has(bag.getWNum1())) {
+            map.put("wnum1", bag.getWNum1());
+        }
+        if (bag.getZone() != null) {
+            map.put("zone", bag.getZone());
+            int offset = TimeZoneConverter.dateZoneToDiff(bag.getZone());
+            if (offset != Integer.MIN_VALUE) {
+                map.put("offset", offset);
+            }
+        }
+        if (has(bag.getSecFraction())) {
+            map.put("sec_fraction", newRationalCanonicalize(context, bag.getSecFraction(), (long)Math.pow(10, bag.getSecFractionSize())));
+        }
+        if (bag.hasSeconds()) {
+            if (has(bag.getSecondsSize())) {
+                map.put("seconds", newRationalCanonicalize(context, bag.getSeconds(), (long) Math.pow(10, bag.getSecondsSize())));
+            }
+            else {
+                map.put("seconds", bag.getSeconds());
+            }
+        }
+        if (has(bag.getMerid())) {
+            map.put("_merid", bag.getMerid());
+        }
+        if (has(bag.getCent())) {
+            map.put("_cent", bag.getCent());
+        }
+        if (bag.getLeftover() != null) {
+            map.put("leftover", bag.getLeftover());
+        }
+
+        return map;
+    }
+}

data/embulk-jruby-strptime/src/main/java/org/embulk/spi/time/StrptimeFormat.java

@@ -0,0 +1,53 @@
+package org.embulk.spi.time;
+
+/**
+ * This class is ported from org.jruby.util.RubyDateFormatter.Format in JRuby
+ * 9.1.5.0 and modified for StrptimeParser under EPL.
+ * @see <a href="https://github.com/jruby/jruby/blob/036ce39f0476d4bd718e23e64caff36bb50b8dbc/core/src/main/java/org/jruby/util/RubyDateFormatter.java>RubyDateFormatter.java</a>.
+ *
+ * TODO
+ * This class is tentatively required for {@code StrptimeParser} class.
+ * The {@code StrptimeParser} and {@code RubyDateParser} will be merged into JRuby
+ * (jruby/jruby#4591). embulk-jruby-strptime is removed when Embulk start using
+ * the JRuby that bundles embulk-jruby-strptime.
+ */
+enum StrptimeFormat
+{
+    FORMAT_STRING, // raw string, no formatting
+    FORMAT_SPECIAL, // composition of other formats
+
+    FORMAT_WEEK_LONG, // %A
+    FORMAT_WEEK_SHORT, // %a
+    FORMAT_MONTH_LONG, // %B
+    FORMAT_MONTH_SHORT, // %b, %h
+    FORMAT_CENTURY, // %C
+    FORMAT_DAY, // %d
+    FORMAT_DAY_S, // %e
+    FORMAT_WEEKYEAR, // %G
+    FORMAT_WEEKYEAR_SHORT, // %g
+    FORMAT_HOUR, // %H
+    FORMAT_HOUR_M, // %I
+    FORMAT_DAY_YEAR, // %j
+    FORMAT_HOUR_BLANK, // %k
+    FORMAT_MILLISEC, // %L
+    FORMAT_HOUR_S, // %l
+    FORMAT_MINUTES, // %M
+    FORMAT_MONTH, // %m
+    FORMAT_NANOSEC, // %N
+    FORMAT_MERIDIAN_LOWER_CASE, // %P
+    FORMAT_MERIDIAN, // %p
+    FORMAT_MICROSEC_EPOCH, // %Q Only for Date/DateTime from here
+    FORMAT_SECONDS, // %S
+    FORMAT_EPOCH, // %s
+    FORMAT_WEEK_YEAR_S, // %U
+    FORMAT_DAY_WEEK2, // %u
+    FORMAT_WEEK_WEEKYEAR, // %V
+    FORMAT_WEEK_YEAR_M, // %W
+    FORMAT_DAY_WEEK, // %w
+    FORMAT_YEAR_LONG, // %Y
+    FORMAT_YEAR_SHORT, // %y
+
+    FORMAT_COLON_ZONE_OFF, // %z, %:z, %::z, %:::z must be given number of colons as data
+
+    FORMAT_ZONE_ID; // %Z Change between Time and Date
+}