embulk-filter-timestamp_format 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0086188ce09565733308b14cebcfd784df0a6a4e
-  data.tar.gz: 8d974fbc9276c45f07541f62f79f725b375f2d89
+  metadata.gz: 81e7bf1b428ed0d48164dbaaec74a2080be029be
+  data.tar.gz: 7c8e45fcc5cf72d8ae25eb973382e74bf3b16420
 SHA512:
-  metadata.gz: fcbe6f58659fa857ddf7237aa9534bcb392ff21d60e1e31abe267414b570a93b44bd347839361a207add780306ceb682e6e3f14e93181b3b1741fd497078bf86
-  data.tar.gz: 2b13d3aab703ba5dfd53f41587d20322595fee6b3af5603f8744c4955b1b32ce2525d022a46bda85d5567de8a687ef1718e3aa064094d278a20406fdf94db6c0
+  metadata.gz: fbae3c05eb9a2808adab605035e5ad432f9fd61df48b5b077565fcc6372a7e3413ec4ea499ddd98b9ebfbbee3f1073f7fa4a73b4743b238c9532905ee60afcfb
+  data.tar.gz: b3760fc88ea943d92edc776ef400563affe983718e7d66854e7093ba7a25983d9395ac3a46642ed430c5ccde4dc3845437e10145d7e4d2385e6b722fb021c311

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.8 (2016-05-10)
+
+Enhancements:
+
+* Support nano second resolution for Java parser
+
 # 0.1.7 (2016-05-09)
 
 Enhancements:

data/README.md

@@ -99,9 +99,23 @@ Benchmark test sets are available at [./bench](./bench).  In my environment (Mac
 * jruby parser / java formatter: 64.52s
 * jruby parser / jruby formatter: 65.06s
 
-**NOTICE:**
+## Nano Resolution
 
-* JRuby parser has micro second resolution, but Java parser (Joda-Time) has only milli second resolution
+JRuby parser has micro second resolution. Java (Joda-Time) parser has milli second resolution (although Java8's DateTimeFormatter supports nano second resolution)
+
+Nano second resolution is partially supported by this plugin itself. Use parser format `nnnnnnnnn` for Java parser as
+
+```
+yyyy-MM-dd HH:mm:ss.nnnnnnnnn z
+```
+
+This plugin finds places of nano second from texts with regular expression `\.(\d+)`.
+
+For formatter, you can use jruby formatter as
+
+```
+%Y-%m-%d %H:%M:%S.%N %z
+```
 
 ## ToDo
 

data/bench/config_java.yml

@@ -8,6 +8,6 @@ in:
 filters:
   - type: timestamp_format
     columns:
-      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSS"], to_format: "yyyy-MM-dd"}
+      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSSSSSSSS"], to_format: "yyyy-MM-dd"}
 out:
   type: "null"

data/bench/config_jruby_formatter.yml

@@ -8,6 +8,6 @@ in:
 filters:
   - type: timestamp_format
     columns:
-      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSS"], to_format: "%Y-%m-%d"}
+      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.SSSSSSSSS"], to_format: "%Y-%m-%d"}
 out:
   type: "null"

data/bench/config_nano.yml

@@ -0,0 +1,13 @@
+in:
+  type: file
+  path_prefix: bench/dummy
+  parser:
+    type: csv
+    columns:
+      - {name: timestamp, type: string}
+filters:
+  - type: timestamp_format
+    columns:
+      - {name: timestamp, from_format: ["yyyy-MM-dd hh:mm:ss.nnnnnnnnn"], to_format: "%Y-%m-%d"}
+out:
+  type: "null"

data/bench/gen_dummy.rb

@@ -1,5 +1,5 @@
 File::open('bench/dummy.csv', 'w') { |f|
   (1..1000000).each {
-    f.puts(Time.now.strftime('%Y-%m-%d %H:%M:%S.%L'))
+    f.puts(Time.now.strftime('%Y-%m-%d %H:%M:%S.%9N'))
   }
 }

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.7"
+version = "0.1.8"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/string.csv

@@ -3,4 +3,12 @@
 2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00,2015-07-13 00:00:00
 2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC,2015-07-12 16:00:00 UTC
 2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC,2015-07-12 16:00:00.1 UTC
+2015-07-12 16:00:00.12 UTC,2015-07-12 16:00:00.12 UTC,2015-07-12 16:00:00.12 UTC,2015-07-12 16:00:00.12 UTC
+2015-07-12 16:00:00.123 UTC,2015-07-12 16:00:00.123 UTC,2015-07-12 16:00:00.123 UTC,2015-07-12 16:00:00.123 UTC
+2015-07-12 16:00:00.1234 UTC,2015-07-12 16:00:00.1234 UTC,2015-07-12 16:00:00.1234 UTC,2015-07-12 16:00:00.1234 UTC
+2015-07-12 16:00:00.12345 UTC,2015-07-12 16:00:00.12345 UTC,2015-07-12 16:00:00.12345 UTC,2015-07-12 16:00:00.12345 UTC
+2015-07-12 16:00:00.123456 UTC,2015-07-12 16:00:00.123456 UTC,2015-07-12 16:00:00.123456 UTC,2015-07-12 16:00:00.123456 UTC
+2015-07-12 16:00:00.1234567 UTC,2015-07-12 16:00:00.1234567 UTC,2015-07-12 16:00:00.1234567 UTC,2015-07-12 16:00:00.1234567 UTC
+2015-07-12 16:00:00.12345678 UTC,2015-07-12 16:00:00.12345678 UTC,2015-07-12 16:00:00.12345678 UTC,2015-07-12 16:00:00.12345678 UTC
+2015-07-12 16:00:00.123456789 UTC,2015-07-12 16:00:00.123456789 UTC,2015-07-12 16:00:00.123456789 UTC,2015-07-12 16:00:00.123456789 UTC
 

data/example/string.yml

@@ -11,7 +11,7 @@ in:
 filters:
   - type: timestamp_format
     default_from_timezone: "Asia/Taipei"
-    default_from_timestamp_format: ["%Y-%m-%d", "%Y-%m-%d %z", "%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S"]
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z", "%Y-%m-%d %H:%M:%S", "%Y-%m-%d %z", "%Y-%m-%d"]
     default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:

data/example/string_java.yml

@@ -11,7 +11,7 @@ in:
 filters:
   - type: timestamp_format
     default_from_timezone: "Asia/Taipei"
-    default_from_timestamp_format: ["yyyy-MM-dd", "yyyy-MM-dd z", "yyyy-MM-dd HH:mm:ss.S z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
+    default_from_timestamp_format: ["yyyy-MM-dd", "yyyy-MM-dd z", "yyyy-MM-dd HH:mm:ss.SSSSSSSSS z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
     default_to_timezone: "Asia/Taipei"
     default_to_timestamp_format: "yyyy-MM-dd HH:mm:ss.SSS Z"
     columns:

data/example/string_nano.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: example/string.csv
+  parser:
+    type: csv
+    columns:
+      - {name: string1, type: string}
+      - {name: string2, type: string}
+      - {name: string3, type: string}
+      - {name: string4, type: string}
+filters:
+  - type: timestamp_format
+    default_from_timezone: "Asia/Taipei"
+    default_from_timestamp_format: ["yyyy-MM-dd", "yyyy-MM-dd z", "yyyy-MM-dd HH:mm:ss.nnnnnnnnn z", "yyyy-MM-dd HH:mm:ss z", "yyyy-MM-dd HH:mm:ss"]
+    default_to_timezone: "Asia/Taipei"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
+    columns:
+      - {name: string1}
+      - {name: string2, type: timestamp}
+      - {name: string3, type: long, to_unit: ms}
+      - {name: string4, type: double, to_unit: ms}
+out:
+  type: "null"

data/src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java

@@ -22,6 +22,8 @@ import org.jruby.embed.ScriptingContainer;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Locale;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import org.joda.time.format.DateTimeFormat;
 
@@ -48,7 +50,9 @@ public class TimestampParser {
 
     private final List<JRubyTimeParserHelper> jrubyParserList = new ArrayList<>();
     private final List<DateTimeFormatter> javaParserList = new ArrayList<>();
+    private final List<Boolean> handleNanoResolutionList = new ArrayList<>();
     private final DateTimeZone defaultFromTimeZone;
+    private final Pattern nanoSecPattern = Pattern.compile("\\.(\\d+)");
 
     TimestampParser(PluginTask task) {
         this(task.getJRuby(), task.getDefaultFromTimestampFormat(), task.getDefaultFromTimeZone());
@@ -69,8 +73,18 @@ public class TimestampParser {
                 JRubyTimeParserHelper helper = (JRubyTimeParserHelper) helperFactory.newInstance(format, 1970, 1, 1, 0, 0, 0, 0);  // TODO default time zone
                 this.jrubyParserList.add(helper);
             } else {
-                DateTimeFormatter parser = DateTimeFormat.forPattern(format).withLocale(Locale.ENGLISH).withZone(defaultFromTimeZone);
-                this.javaParserList.add(parser);
+                // special treatment for nano resolution. n is not originally supported by Joda-Time
+                if (format.contains("n")) {
+                    this.handleNanoResolutionList.add(true);
+                    String newFormat = format.replaceAll("n", "S");
+                    DateTimeFormatter parser = DateTimeFormat.forPattern(newFormat).withLocale(Locale.ENGLISH).withZone(defaultFromTimeZone);
+                    this.javaParserList.add(parser);
+                }
+                else {
+                    this.handleNanoResolutionList.add(false);
+                    DateTimeFormatter parser = DateTimeFormat.forPattern(format).withLocale(Locale.ENGLISH).withZone(defaultFromTimeZone);
+                    this.javaParserList.add(parser);
+                }
             }
         }
         this.defaultFromTimeZone = defaultFromTimeZone;
@@ -127,23 +141,48 @@ public class TimestampParser {
     }
 
     private Timestamp javaParse(String text) throws IllegalArgumentException {
-        DateTime dateTime = null;
+        long msec = -1;
+        long nsec = -1;
+        Boolean handleNanoResolution = false;
         IllegalArgumentException exception = null;
 
-        for (DateTimeFormatter parser : javaParserList) {
+        for (int i = 0; i < javaParserList.size(); i++) {
+            DateTimeFormatter parser = javaParserList.get(i);
+            handleNanoResolution = handleNanoResolutionList.get(i);
             try {
-                dateTime = parser.parseDateTime(text);
+                if (handleNanoResolution) {
+                    nsec = parseNano(text);
+                }
+                DateTime dateTime = parser.parseDateTime(text);
+                msec = dateTime.getMillis(); // NOTE: milli second resolution
                 break;
             } catch (IllegalArgumentException ex) {
                 exception = ex;
             }
         }
-        if (dateTime == null) {
+        if (msec == -1) {
             throw exception;
         }
-        long msec = dateTime.getMillis(); // NOTE: milli second resolution
 
-        long nanoAdjustment = msec * 1000000;
-        return Timestamp.ofEpochSecond(0, nanoAdjustment);
+        if (handleNanoResolution) {
+            long sec = msec / 1000;
+            return Timestamp.ofEpochSecond(sec, nsec);
+        }
+        else {
+            long nanoAdjustment = msec * 1000000;
+            return Timestamp.ofEpochSecond(0, nanoAdjustment);
+        }
+    }
+
+    private long parseNano(String text) {
+        long nsec = -1;
+        Matcher m = nanoSecPattern.matcher(text);
+        if (m.find()) {
+            //String nanoStr = String.format("%-9s", m.group(1)).replace(" ", "0");
+            //nsec = Long.parseLong(nanoStr);
+            String nanoStr = m.group(1);
+            nsec = Long.parseLong(nanoStr) * (long) Math.pow(10, 9 - nanoStr.length());
+        }
+        return nsec;
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-timestamp_format
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-09 00:00:00.000000000 Z
+date: 2016-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -54,6 +54,7 @@ files:
 - bench/config_jruby.yml
 - bench/config_jruby_formatter.yml
 - bench/config_jruby_parser.yml
+- bench/config_nano.yml
 - bench/gen_dummy.rb
 - build.gradle
 - config/checkstyle/checkstyle.xml
@@ -74,6 +75,7 @@ files:
 - example/string.csv
 - example/string.yml
 - example/string_java.yml
+- example/string_nano.yml
 - example/timestamp.csv
 - example/timestamp.yml
 - gradle/wrapper/gradle-wrapper.jar
@@ -95,7 +97,7 @@ files:
 - src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java
 - src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java
 - src/test/java/org/embulk/filter/timestamp_format/TestTimestampUnit.java
-- classpath/embulk-filter-timestamp_format-0.1.7.jar
+- classpath/embulk-filter-timestamp_format-0.1.8.jar
 homepage: https://github.com/sonots/embulk-filter-timestamp_format
 licenses:
 - MIT