embulk-filter-timestamp_format 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb84426fbf7dfbaac21925ece7475f322c7c2010
-  data.tar.gz: 2c5accec8470864588c20d62c42626a89a584bab
+  metadata.gz: 0086188ce09565733308b14cebcfd784df0a6a4e
+  data.tar.gz: 8d974fbc9276c45f07541f62f79f725b375f2d89
 SHA512:
-  metadata.gz: d77f18cde0d2a3626198d1e94aa1c7e1b6c47d3c8572c9397cf1c5ec8a4ec808fcb9dea636339c0e838e00f418cefd290d92d77b44669a6df31c1c3e2fd8b9bb
-  data.tar.gz: c3464272e96b244c782b65e8e536a1f64fc488646958c58d04a068550c85d718f137081662ce5fd7fe1370aa6788bda4b8598a7e0e606b7a00e6811e096177a5
+  metadata.gz: fcbe6f58659fa857ddf7237aa9534bcb392ff21d60e1e31abe267414b570a93b44bd347839361a207add780306ceb682e6e3f14e93181b3b1741fd497078bf86
+  data.tar.gz: 2b13d3aab703ba5dfd53f41587d20322595fee6b3af5603f8744c4955b1b32ce2525d022a46bda85d5567de8a687ef1718e3aa064094d278a20406fdf94db6c0

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+# 0.1.7 (2016-05-09)
+
+Enhancements:
+
+* Use Joda-Time DateTimeFormat instead of SimpleDateFormat for Java timestamp parser/formatter
+  * to be thread-safe
+  * to fix ss.SSS resolves 1.1 as 1.001 seconds wrongly
+
 # 0.1.6 (2016-05-01)
 
 Enhancements:

data/README.md

@@ -8,7 +8,7 @@ A filter plugin for Embulk to change timestamp format
 
 - **columns**: columns to retain (array of hash)
   - **name**: name of column (required)
-  - **type**: type to cast (string, timestamp, long (unixtimestamp), double (unixtimestamp), default is string)
+  - **type**: type to cast, choose one of `string`, `timestamp`, `long` (unixtimestamp), `double` (unixtimestamp) (string, default is `string`)
   - **from_format**: specify the format of the input string (array of strings, default is default_from_timestamp_format)
   - **from_timezone**: specify the timezone of the input string (string, default is default_from_timezone)
   - **to_format**: specify the format of the output string (string, default is default_to_timestamp_format)
@@ -19,8 +19,8 @@ A filter plugin for Embulk to change timestamp format
 - **default_from_timezone**: default timezone for the input string (string, default is `UTC`)
 - **default_to_timestamp_format**: default timestamp format for the output string (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
 - **default_to_timezone**: default timezone for the output string (string, default is `UTC`)
-- **default_from_timetamp_unit**: default time unit such as second, ms, us, ns for the input unixtimestamp (string, default is `second`)
-- **default_to_timetamp_unit**: default time unit such as second, ms, us, ns for the output unixtimestamp (string, default is `second`)
+- **default_from_timestamp_unit**: default time unit such as `second`, `ms`, `us`, `ns` for the input unixtimestamp (string, default is `second`)
+- **default_to_timestamp_unit**: default time unit such as `second`, `ms`, `us`, `ns` for the output unixtimestamp (string, default is `second`)
 - **stop_on_invalid_record**: stop bulk load transaction if a invalid record is found (boolean, default is `false`)
 
 ## Example
@@ -37,33 +37,35 @@ in:
   type: file
   path_prefix: example/example.jsonl
   parser:
-    type: jsonl
+    type: jsonl # not json parser
     columns:
     - {name: timestamp, type: string}
     - {name: nested, type: json}
 filters:
   - type: timestamp_format
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]
     default_to_timezone: "Asia/Tokyo"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:
-      - {name: timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
-      - {name: $.nested.timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: timestamp, type: long, to_unit: ms}
+      - {name: $.nested.timestamp}
+out:
   type: stdout
 ```
 
 Output will be as:
 
 ```
-{"timestamp":"2015-07-13 00:00:00.0","nested":{"timestamp":"2015-07-13 00:00:00.0}}
-{"timestamp":"2015-07-13 00:00:00.1","nested":{"timestamp":"2015-07-13 00:00:00.1}}
+{"timestamp":1436713200000,"nested":{"timestamp":"2015-07-13 00:00:00.0}}
+{"timestamp":1436713200100,"nested":{"timestamp":"2015-07-13 00:00:00.1}}
 ```
 
 See [./example](./example) for more examples.
 
-## Timestamp Parser/Formatter Performance Issue
+## JRuby Timestamp Parser/Formatter Performance Issue
 
 Embulk's timestamp parser/formatter originally uses jruby implementation, but it is slow.
-To improve performance, this plugin also supports Java's [SimpleDateFormat](https://docs.oracle.com/javase/jp/6/api/java/text/SimpleDateFormat.html) format as:
+To improve performance, this plugin also supports Java's Joda-Time [DateTimeFormat](http://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html) format as:
 
 ```yaml
 in:
@@ -76,12 +78,11 @@ in:
     - {name: nested, type: json}
 filters:
   - type: timestamp_format
-    default_from_timezone: "Asia/Taipei"
     default_from_timestamp_format: ["yyyy-MM-dd HH:mm:ss.SSS z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
     default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss.SSS Z"
     columns:
-      - {name: timestamp}
+      - {name: timestamp, type: long, to_unit: ms}
       - {name: $.nested.timestamp}
 out:
   type: stdout
@@ -93,13 +94,14 @@ If format strings contain `%`, jruby parser/formatter is used. Otherwirse, java
 
 Benchmark test sets are available at [./bench](./bench).  In my environment (Mac Book Pro), for 1000000 timestamps:
 
-* jruby parser/formatter: 65.06s
-* java parser/formatter: 1.3s
+* java parser / java formatter: 1.3s
+* java parser / jruby formatter: 1.4s
+* jruby parser / java formatter: 64.52s
+* jruby parser / jruby formatter: 65.06s
 
 **NOTICE:**
 
-* JRuby parser has micro second resolution, but Java parser (SimpleDateFormat) has only milli second resolution
-* `S` requires three digits always. For example, `yyyy-MM-dd HH:mm::ss.S` for `2015-12-17 01:02:03.1` gives 001 milli seconds wrongly, but it is the specification of SimpleDateFormat.
+* JRuby parser has micro second resolution, but Java parser (Joda-Time) has only milli second resolution
 
 ## ToDo
 

data/bench/config_java.yml

@@ -9,6 +9,5 @@ filters:
   - type: timestamp_format
     columns:
       - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSS"], to_format: "yyyy-MM-dd"}
-
 out:
   type: "null"

data/bench/config_jruby_formatter.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: bench/dummy
+  parser:
+    type: csv
+    columns:
+      - {name: timestamp, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSS"], to_format: "%Y-%m-%d"}
+out:
+  type: "null"

data/bench/config_jruby_parser.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: bench/dummy
+  parser:
+    type: csv
+    columns:
+      - {name: timestamp, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N"], to_format: "yyyy-MM-dd"}
+out:
+  type: "null"

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.6"
+version = "0.1.7"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/example.yml

@@ -8,7 +8,7 @@ filters:
     default_to_timezone: "Asia/Tokyo"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:
-      - {name: "$.record.timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: "$.record.timestamp", type: long, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"], to_unit: ms}
       - {name: "$.record.nested.nested[0].timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
 out:
   type: "null"

data/example/string.csv

@@ -1,3 +1,5 @@
+2015-07-13,2015-07-13,2015-07-13,2015-07-13
+2015-07-13 UTC,2015-07-13 UTC,2015-07-13 UTC,2015-07-13 UTC
 2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00
 2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC
 2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC

data/example/string.yml

@@ -11,7 +11,7 @@ in:
 filters:
   - type: timestamp_format
     default_from_timezone: "Asia/Taipei"
-    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S"]
+    default_from_timestamp_format: ["%Y-%m-%d", "%Y-%m-%d %z", "%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S"]
     default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:

data/example/string_java.yml

@@ -11,7 +11,7 @@ in:
 filters:
   - type: timestamp_format
     default_from_timezone: "Asia/Taipei"
-    default_from_timestamp_format: ["yyyy-MM-dd HH:mm:ss.S z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"] # SSS must be three digit ...
+    default_from_timestamp_format: ["yyyy-MM-dd", "yyyy-MM-dd z", "yyyy-MM-dd HH:mm:ss.S z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
     default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss.SSS Z"
     columns:

data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java

@@ -15,9 +15,9 @@ import org.joda.time.DateTimeZone;
 import org.jruby.embed.ScriptingContainer;
 import org.jruby.util.RubyDateFormat;
 
-import java.text.SimpleDateFormat;
-import java.util.Date;
 import java.util.Locale;
+import org.joda.time.format.DateTimeFormat;
+import org.joda.time.format.DateTimeFormatter;
 
 public class TimestampFormatter
 {
@@ -44,7 +44,7 @@ public class TimestampFormatter
     }
 
     private final RubyDateFormat jrubyFormatter;
-    private final SimpleDateFormat javaFormatter;
+    private final DateTimeFormatter javaFormatter;
     private final DateTimeZone toTimeZone;
 
     public TimestampFormatter(PluginTask task, Optional<? extends TimestampColumnOption> columnOption)
@@ -67,8 +67,7 @@ public class TimestampFormatter
         }
         else {
             this.jrubyFormatter = null;
-            this.javaFormatter = new SimpleDateFormat(format, Locale.ENGLISH);
-            javaFormatter.setTimeZone(toTimeZone.toTimeZone());
+            this.javaFormatter = DateTimeFormat.forPattern(format).withLocale(Locale.ENGLISH).withZone(toTimeZone);
         }
     }
 
@@ -108,6 +107,6 @@ public class TimestampFormatter
     private String javaFormat(Timestamp value)
     {
         long milliSecond = value.getEpochSecond() * 1000 + value.getNano() / 1000000;
-        return javaFormatter.format(milliSecond);
+        return javaFormatter.print(milliSecond);
     }
 }

data/src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java

@@ -14,14 +14,16 @@ import org.embulk.spi.time.Timestamp;
 import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
 
 import org.embulk.spi.time.TimestampParseException;
+import org.joda.time.DateTime;
 import org.joda.time.DateTimeZone;
+import org.joda.time.format.DateTimeFormatter;
 import org.jruby.embed.ScriptingContainer;
 
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.TimeZone;
+import java.util.Locale;
+
+import org.joda.time.format.DateTimeFormat;
 
 public class TimestampParser {
     public interface Task {
@@ -44,8 +46,8 @@ public class TimestampParser {
         Optional<List<String>> getFromFormat();
     }
 
-    private final List<JRubyTimeParserHelper> jrubyParserList = new ArrayList<JRubyTimeParserHelper>();
-    private final List<SimpleDateFormat> javaParserList = new ArrayList<SimpleDateFormat>();
+    private final List<JRubyTimeParserHelper> jrubyParserList = new ArrayList<>();
+    private final List<DateTimeFormatter> javaParserList = new ArrayList<>();
     private final DateTimeZone defaultFromTimeZone;
 
     TimestampParser(PluginTask task) {
@@ -60,14 +62,14 @@ public class TimestampParser {
 
     public TimestampParser(ScriptingContainer jruby, List<String> formatList, DateTimeZone defaultFromTimeZone) {
         JRubyTimeParserHelperFactory helperFactory = (JRubyTimeParserHelperFactory) jruby.runScriptlet("Embulk::Java::TimeParserHelper::Factory.new");
+
         // TODO get default current time from ExecTask.getExecTimestamp
         for (String format : formatList) {
             if (format.contains("%")) {
                 JRubyTimeParserHelper helper = (JRubyTimeParserHelper) helperFactory.newInstance(format, 1970, 1, 1, 0, 0, 0, 0);  // TODO default time zone
                 this.jrubyParserList.add(helper);
             } else {
-                SimpleDateFormat parser = new SimpleDateFormat(format);
-                parser.setTimeZone(defaultFromTimeZone.toTimeZone());
+                DateTimeFormatter parser = DateTimeFormat.forPattern(format).withLocale(Locale.ENGLISH).withZone(defaultFromTimeZone);
                 this.javaParserList.add(parser);
             }
         }
@@ -78,7 +80,7 @@ public class TimestampParser {
         return defaultFromTimeZone;
     }
 
-    public Timestamp parse(String text) throws TimestampParseException, ParseException {
+    public Timestamp parse(String text) throws TimestampParseException, IllegalArgumentException {
         if (!jrubyParserList.isEmpty()) {
             return jrubyParse(text);
         } else if (!javaParserList.isEmpty()) {
@@ -124,21 +126,22 @@ public class TimestampParser {
         return Timestamp.ofEpochSecond(sec, usec * 1000);
     }
 
-    private Timestamp javaParse(String text) throws ParseException {
-        long msec = -1;
-        ParseException exception = null;
+    private Timestamp javaParse(String text) throws IllegalArgumentException {
+        DateTime dateTime = null;
+        IllegalArgumentException exception = null;
 
-        for (SimpleDateFormat parser : javaParserList) {
+        for (DateTimeFormatter parser : javaParserList) {
             try {
-                msec = parser.parse(text).getTime(); // NOTE: milli second resolution
+                dateTime = parser.parseDateTime(text);
                 break;
-            } catch (ParseException ex) {
+            } catch (IllegalArgumentException ex) {
                 exception = ex;
             }
         }
-        if (msec == -1) {
+        if (dateTime == null) {
             throw exception;
         }
+        long msec = dateTime.getMillis(); // NOTE: milli second resolution
 
         long nanoAdjustment = msec * 1000000;
         return Timestamp.ofEpochSecond(0, nanoAdjustment);

data/src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java

@@ -7,8 +7,6 @@ import org.embulk.spi.DataException;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampParseException;
 
-import java.text.ParseException;
-
 public class StringCast
 {
     private StringCast() {}
@@ -27,7 +25,7 @@ public class StringCast
         catch (TimestampParseException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
-        catch (ParseException ex) {
+        catch (IllegalArgumentException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
     }
@@ -40,7 +38,7 @@ public class StringCast
         catch (TimestampParseException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
-        catch (ParseException ex) {
+        catch (IllegalArgumentException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
     }
@@ -54,7 +52,7 @@ public class StringCast
         catch (TimestampParseException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
-        catch (ParseException ex) {
+        catch (IllegalArgumentException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
     }
@@ -68,7 +66,7 @@ public class StringCast
         catch (TimestampParseException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
-        catch (ParseException ex) {
+        catch (IllegalArgumentException ex) {
             throw new DataException(buildErrorMessage(value), ex);
         }
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-timestamp_format
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-01 00:00:00.000000000 Z
+date: 2016-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,8 @@ files:
 - README.md
 - bench/config_java.yml
 - bench/config_jruby.yml
+- bench/config_jruby_formatter.yml
+- bench/config_jruby_parser.yml
 - bench/gen_dummy.rb
 - build.gradle
 - config/checkstyle/checkstyle.xml
@@ -93,7 +95,7 @@ files:
 - src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java
 - src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java
 - src/test/java/org/embulk/filter/timestamp_format/TestTimestampUnit.java
-- classpath/embulk-filter-timestamp_format-0.1.6.jar
+- classpath/embulk-filter-timestamp_format-0.1.7.jar
 homepage: https://github.com/sonots/embulk-filter-timestamp_format
 licenses:
 - MIT