embulk-filter-timestamp_format 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c06f4d7efebebe0e2abb454b7f6545e6ea843026
-  data.tar.gz: d98aceb871fe643ed6afe46501758d15884ee124
+  metadata.gz: 7d7569b8adc1db79b292e271214f852fb080151b
+  data.tar.gz: df0c01a5893dc4a4bbb1f1228e3d72b031e59f93
 SHA512:
-  metadata.gz: d8aae59c031cd582e3df88466417822e1f952ec9573d1df8fe189109496e909c9ca71d6e448d544ac8ffa64611ec2689651fccb7caa4e5de8559ff3c30bc6213
-  data.tar.gz: 938432021acdece8554c884e033f5e4f2bcac15d2c0407fff3134f8757f84f52201e82f14ac46a4d954c900382d3d174e79ee04b94dc76848867a1b4ae4afe9e
+  metadata.gz: d81f4f2df4775444b5608432a2451158453f768cf6687188afae1169ea5eb15c699141d670987e73139c14c5ae8bbeb2122fbfcb6f73c89f5e98a425db8f2519
+  data.tar.gz: 5ecdc2f30763b7768fd1e9176c2c6b01fdafbd214f9191885e79de7f333303a9a7b86054a7c10acbf87ed458db5cab5d3b2c4871115198b829f65b8c36d855cc

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+# 0.1.5 (2016-04-29)
+
+Enhancements:
+
+* Support to cast from/into timestamp
+* Support to cast into long/double (unixtimesatmp)
+
 # 0.1.4 (2016-04-26)
 
 Enhancements:

data/README.md

@@ -2,20 +2,21 @@
 
 [![Build Status](https://secure.travis-ci.org/sonots/embulk-filter-timestamp_format.png?branch=master)](http://travis-ci.org/sonots/embulk-filter-timestamp_format)
 
-A filter plugin for Embulk to change timesatmp format
+A filter plugin for Embulk to change timestamp format
 
 ## Configuration
 
 - **columns**: columns to retain (array of hash)
-  - **name**: name of column, must be a string column (required)
-  - **from_format**: specify the format of the input timestamp (array of strings, default is default_from_format)
-  - **from_timezone**: specify the timezone of the input timestamp (string, default is default_from_timezone)
-  - **to_format**: specify the format of the output timestamp (string, default is default_to_format)
-  - **to_timezone**: specify the timezone of the output timestamp (string, default is default_to_timezone)
-- **default_from_format**: default timestamp format for the input timestamp columns (array of strings, default is `["%Y-%m-%d %H:%M:%S.%N %z"]`)
-- **default_from_timezone**: default timezone for the input timestamp columns (string, default is `UTC`)
-- **default_to_format**: default timestamp format for the output timestamp columns (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
-- **default_to_timezone**: default timezone for the output timestamp olumns (string, default is `UTC`)
+  - **name**: name of column (required)
+  - **type**: type to cast (string, timestamp, long (unixtimestamp), double (unixtimestamp), default is string)
+  - **from_format**: specify the format of the input string (array of strings, default is default_from_timestamp_format)
+  - **from_timezone**: specify the timezone of the input string (string, default is default_from_timezone)
+  - **to_format**: specify the format of the output string (string, default is default_to_timestamp_format)
+  - **to_timezone**: specify the timezone of the output string (string, default is default_to_timezone)
+- **default_from_timestamp_format**: default timestamp format for the input string (array of strings, default is `["%Y-%m-%d %H:%M:%S.%N %z"]`)
+- **default_from_timezone**: default timezone for the input string (string, default is `UTC`)
+- **default_to_timestamp_format**: default timestamp format for the output string (string, default is `%Y-%m-%d %H:%M:%S.%N %z`)
+- **default_to_timezone**: default timezone for the output string (string, default is `UTC`)
 * **stop_on_invalid_record**: stop bulk load transaction if a invalid record is found (boolean, default is `false)
 
 ## Example
@@ -23,8 +24,8 @@ A filter plugin for Embulk to change timesatmp format
 Say example.jsonl is as follows (this is a typical format which Exporting BigQuery table outputs):
 
 ```
-{"timestamp":"2015-07-12 15:00:00 UTC","record":{"timestamp":"2015-07-12 15:00:00 UTC"}}
-{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}
+{"timestamp":"2015-07-12 15:00:00 UTC","nested":{"timestamp":"2015-07-12 15:00:00 UTC"}}
+{"timestamp":"2015-07-12 15:00:00.1 UTC","nested":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}
 ```
 
 ```yaml
@@ -35,27 +36,28 @@ in:
     type: jsonl
     columns:
     - {name: timestamp, type: string}
-    - {name: record, type: json}
+    - {name: nested, type: json}
 filters:
   - type: timestamp_format
     default_to_timezone: "Asia/Tokyo"
-    default_to_format: "%Y-%m-%d %H:%M:%S.%N"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
     columns:
       - {name: timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
-      - {name: record.timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: $.nested.timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
   type: stdout
 ```
 
 Output will be as:
 
 ```
-{"timestamp":"2015-07-13 00:00:00.0","record":{"timestamp":"2015-07-13 00:00:00.0}}
-{"timestamp":"2015-07-13 00:00:00.1","record":{"timestamp":"2015-07-13 00:00:00.1}}
+{"timestamp":"2015-07-13 00:00:00.0","nested":{"timestamp":"2015-07-13 00:00:00.0}}
+{"timestamp":"2015-07-13 00:00:00.1","nested":{"timestamp":"2015-07-13 00:00:00.1}}
 ```
 
+See [./example](./example) for more examples.
+
 ## ToDo
 
-* Currently, input must be a String column and output will be a String column. But, support Timestamp column (input / output)
 * Write test
 
 ## Development
@@ -63,9 +65,8 @@ Output will be as:
 Run example:
 
 ```
-$ embulk gem install embulk-parser-jsonl
 $ ./gradlew classpath
-$ embulk run -I lib example/example.yml
+$ embulk preview -I lib example/example.yml
 ```
 
 Run test:

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -72,7 +72,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
-  spec.add_development_dependency 'embulk-parser-jsonl'
 end
 /$)
     }

data/example/example.yml

@@ -1,18 +1,22 @@
 in:
   type: file
-  path_prefix: example/example.jsonl
+  path_prefix: example/string_example.csv
   parser:
-    type: jsonl
+    type: csv
     columns:
-    - {name: timestamp, type: string}
-    - {name: record, type: json}
-    - {name: ignore_record, type: json}
+      - {name: string1, type: string}
+      - {name: string2, type: string}
+      - {name: string3, type: string}
+      - {name: string4, type: string}
 filters:
   - type: timestamp_format
     default_to_timezone: "Asia/Tokyo"
-    default_to_format: "%Y-%m-%d %H:%M:%S.%N"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]
     columns:
-      - {name: timestamp, from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
-      - {name: "$.record.record[0].timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: string1}
+      - {name: string2, type: timestamp}
+      - {name: string3, type: long}
+      - {name: string4, type: double}
 out:
-  type: stdout
+  type: "null"

data/example/json_example.jsonl

@@ -0,0 +1,2 @@
+{"timestamp":"2015-07-12 15:00:00 UTC","nested":{"nested":[{"timestamp":"2015-07-12 15:00:00 UTC"}]},"ignore_nested":{"timestamp":"2015-07-12 15:00:00 UTC"}}
+{"timestamp":"2015-07-12 15:00:00.1 UTC","nested":{"nested":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]},"ignore_nested":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}

data/example/json_example.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: example/json_example.jsonl
+  parser:
+    type: json
+filters:
+  - type: timestamp_format
+    default_to_timezone: "Asia/Tokyo"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
+    columns:
+      - {name: "$.record.timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+      - {name: "$.record.nested.nested[0].timestamp", from_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]}
+out:
+  type: "null"

data/example/string_example.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: example/string_example.csv
+  parser:
+    type: csv
+    columns:
+      - {name: string1, type: string}
+      - {name: string2, type: string}
+      - {name: string3, type: string}
+      - {name: string4, type: string}
+filters:
+  - type: timestamp_format
+    default_to_timezone: "Asia/Tokyo"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
+    default_from_timestamp_format: ["%Y-%m-%d %H:%M:%S.%N %z", "%Y-%m-%d %H:%M:%S %z"]
+    columns:
+      - {name: string1}
+      - {name: string2, type: timestamp}
+      - {name: string3, type: long}
+      - {name: string4, type: double}
+out:
+  type: "null"

data/example/timestamp_example.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: example/timestamp_example.csv
+  parser:
+    type: csv
+    default_timestamp_format: "%Y-%m-%d %H:%M:%S.%N %z"
+    columns:
+      - {name: timestamp1, type: timestamp}
+      - {name: timestamp2, type: timestamp}
+      - {name: timestamp3, type: timestamp}
+      - {name: timestamp4, type: timestamp}
+filters:
+  - type: timestamp_format
+    default_to_timezone: "Asia/Tokyo"
+    default_to_timestamp_format: "%Y-%m-%d %H:%M:%S.%N"
+    columns:
+      - {name: timestamp1}
+      - {name: timestamp2, type: timestamp}
+      - {name: timestamp3, type: long}
+      - {name: timestamp4, type: double}
+out:
+  type: "null"

data/src/main/java/org/embulk/filter/timestamp_format/ColumnCaster.java

@@ -0,0 +1,132 @@
+package org.embulk.filter.timestamp_format;
+
+import org.embulk.filter.timestamp_format.cast.StringCast;
+import org.embulk.filter.timestamp_format.cast.TimestampCast;
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
+import org.joda.time.DateTimeZone;
+import org.msgpack.value.Value;
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+
+public class ColumnCaster
+{
+    private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
+    private final PluginTask task;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final PageReader pageReader;
+    private final PageBuilder pageBuilder;
+    private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<>();
+    private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<>();
+    private final JsonVisitor jsonVisitor;
+
+    ColumnCaster(PluginTask task, Schema inputSchema, Schema outputSchema, PageReader pageReader, PageBuilder pageBuilder)
+    {
+        this.task         = task;
+        this.inputSchema  = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageReader   = pageReader;
+        this.pageBuilder  = pageBuilder;
+
+        buildTimestampParserMap();
+        buildTimestampFormatterMap();
+
+        JsonCaster jsonCaster = new JsonCaster(task, timestampParserMap, timestampFormatterMap);
+        this.jsonVisitor = new JsonVisitor(task, jsonCaster);
+    }
+
+    private void buildTimestampParserMap()
+    {
+        // columnName or jsonPath => TimestampParser
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            TimestampParser parser = getTimestampParser(columnConfig, task);
+            this.timestampParserMap.put(columnConfig.getName(), parser);
+        }
+    }
+
+    private void buildTimestampFormatterMap()
+    {
+        // columnName or jsonPath => TimestampFormatter
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
+            this.timestampFormatterMap.put(columnConfig.getName(), parser);
+        }
+    }
+
+    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
+    {
+        DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
+        List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
+        return new TimestampParser(task.getJRuby(), formatList, timezone);
+    }
+
+    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
+    {
+        String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
+        DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
+        return new TimestampFormatter(task.getJRuby(), format, timezone);
+    }
+
+    public void setFromString(Column outputColumn, String value)
+    {
+        Type outputType = outputColumn.getType();
+        TimestampParser timestampParser = timestampParserMap.get(outputColumn.getName());
+        if (outputType instanceof StringType) {
+            TimestampFormatter timestampFormatter = timestampFormatterMap.get(outputColumn.getName());
+            pageBuilder.setString(outputColumn, StringCast.asString(value, timestampParser, timestampFormatter));
+        }
+        else if (outputType instanceof TimestampType) {
+            pageBuilder.setTimestamp(outputColumn, StringCast.asTimestamp(value, timestampParser));
+        }
+        else if (outputType instanceof LongType) {
+            pageBuilder.setLong(outputColumn, StringCast.asLong(value, timestampParser));
+        }
+        else if (outputType instanceof DoubleType) {
+            pageBuilder.setDouble(outputColumn, StringCast.asDouble(value, timestampParser));
+        }
+        else {
+            assert false;
+        }
+    }
+
+    public void setFromTimestamp(Column outputColumn, Timestamp value)
+    {
+        Type outputType = outputColumn.getType();
+        if (outputType instanceof StringType) {
+            TimestampFormatter timestampFormatter = timestampFormatterMap.get(outputColumn.getName());
+            pageBuilder.setString(outputColumn, TimestampCast.asString(value, timestampFormatter));
+        }
+        else if (outputType instanceof TimestampType) {
+            pageBuilder.setTimestamp(outputColumn, value);
+        }
+        else if (outputType instanceof LongType) {
+            pageBuilder.setLong(outputColumn, TimestampCast.asLong(value));
+        }
+        else if (outputType instanceof DoubleType) {
+            pageBuilder.setDouble(outputColumn, TimestampCast.asDouble(value));
+        }
+        else {
+            assert false;
+        }
+    }
+
+    public void setFromJson(Column outputColumn, Value value)
+    {
+        String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
+        pageBuilder.setJson(outputColumn, jsonVisitor.visit(jsonPath, value));
+    }
+}

data/src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java

@@ -1,251 +1,154 @@
 package org.embulk.filter.timestamp_format;
 
-import com.google.common.base.Throwables;
+import org.embulk.spi.DataException;
 import org.embulk.spi.PageReader;
-import org.msgpack.value.ArrayValue;
-import org.msgpack.value.MapValue;
-import org.msgpack.value.Value;
-import org.msgpack.value.ValueFactory;
+import org.embulk.spi.Schema;
 
-import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
 import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
 
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParseException;
-import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
 
 public class ColumnVisitorImpl
         implements ColumnVisitor
 {
     private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
     private final PluginTask task;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
     private final PageReader pageReader;
     private final PageBuilder pageBuilder;
-    private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<String, TimestampParser>();
-    private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<String, TimestampFormatter>();
-    private final HashSet<String> shouldVisitRecursivelySet = new HashSet<String>();
+    private final HashMap<String, Column> outputColumnMap = new HashMap<>();
+    private final ColumnCaster columnCaster;
 
-    ColumnVisitorImpl(PluginTask task, PageReader pageReader, PageBuilder pageBuilder)
+    ColumnVisitorImpl(PluginTask task, Schema inputSchema, Schema outputSchema,
+                      PageReader pageReader, PageBuilder pageBuilder)
     {
-        this.task        = task;
-        this.pageReader  = pageReader;
-        this.pageBuilder = pageBuilder;
-
-        buildTimestampParserMap();
-        buildTimestampFormatterMap();
-        buildShouldVisitRecursivelySet();;
+        this.task         = task;
+        this.inputSchema  = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageReader   = pageReader;
+        this.pageBuilder  = pageBuilder;
+
+        buildOutputColumnMap();
+        this.columnCaster = new ColumnCaster(task, inputSchema, outputSchema, pageReader, pageBuilder);
     }
 
-    private void buildTimestampParserMap()
+    private void buildOutputColumnMap()
     {
-        // columnName or jsonPath => TimestampParser
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            TimestampParser parser = getTimestampParser(columnConfig, task);
-            this.timestampParserMap.put(columnConfig.getName(), parser); // NOTE: value would be null
+        // columnName => outputColumn
+        for (Column column : outputSchema.getColumns()) {
+            this.outputColumnMap.put(column.getName(), column);
         }
     }
 
-    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
-    {
-        DateTimeZone timezone = columnConfig.getFromTimeZone().or(task.getDefaultFromTimeZone());
-        List<String> formatList = columnConfig.getFromFormat().or(task.getDefaultFromTimestampFormat());
-        return new TimestampParser(task.getJRuby(), formatList, timezone);
-    }
-
-    private void buildTimestampFormatterMap()
+    private interface PageBuildable
     {
-        // columnName or jsonPath => TimestampFormatter
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
-            this.timestampFormatterMap.put(columnConfig.getName(), parser); // NOTE: value would be null
-        }
+        public void run() throws DataException;
     }
 
-    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
+    private void withStopOnInvalidRecord(final PageBuildable op,
+                                         final Column inputColumn, final Column outputColumn) throws DataException
     {
-        String format = columnConfig.getToFormat().or(task.getDefaultToTimestampFormat());
-        DateTimeZone timezone = columnConfig.getToTimeZone().or(task.getDefaultToTimeZone());
-        return new TimestampFormatter(task.getJRuby(), format, timezone);
-    }
-
-
-    private void buildShouldVisitRecursivelySet()
-    {
-        // json partial path => Boolean to avoid unnecessary type: json visit
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            String name = columnConfig.getName();
-            if (!name.startsWith("$.")) {
-                continue;
-            }
-            String[] parts = name.split("\\.");
-            StringBuilder partialPath = new StringBuilder("$");
-            for (int i = 1; i < parts.length; i++) {
-                if (parts[i].contains("[")) {
-                    String[] arrayParts = parts[i].split("\\[");
-                    partialPath.append(".").append(arrayParts[0]);
-                    this.shouldVisitRecursivelySet.add(partialPath.toString());
-                    for (int j = 1; j < arrayParts.length; j++) {
-                        partialPath.append("[").append(arrayParts[j]);
-                        this.shouldVisitRecursivelySet.add(partialPath.toString());
-                    }
-                }
-                else {
-                    partialPath.append(".").append(parts[i]);
-                    this.shouldVisitRecursivelySet.add(partialPath.toString());
-                }
-            }
-        }
-    }
-
-    private boolean shouldVisitRecursively(String name)
-    {
-        return shouldVisitRecursivelySet.contains(name);
-    }
-
-    private Value formatTimestampStringRecursively(PluginTask task, String path, Value value)
-            throws TimestampParseException
-    {
-        if (!shouldVisitRecursively(path)) {
-            return value;
-        }
-        if (value.isArrayValue()) {
-            ArrayValue arrayValue = value.asArrayValue();
-            int size = arrayValue.size();
-            Value[] newValue = new Value[size];
-            for (int i = 0; i < size; i++) {
-                String k = new StringBuilder(path).append("[").append(Integer.toString(i)).append("]").toString();
-                Value v = arrayValue.get(i);
-                newValue[i] = formatTimestampStringRecursively(task, k, v);
-            }
-            return ValueFactory.newArray(newValue, true);
-        }
-        else if (value.isMapValue()) {
-            MapValue mapValue = value.asMapValue();
-            int size = mapValue.size() * 2;
-            Value[] newValue = new Value[size];
-            int i = 0;
-            for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
-                Value k = entry.getKey();
-                Value v = entry.getValue();
-                String newPath = new StringBuilder(path).append(".").append(k.asStringValue().asString()).toString();
-                Value r = formatTimestampStringRecursively(task, newPath, v);
-                newValue[i++] = k;
-                newValue[i++] = r;
-            }
-            return ValueFactory.newMap(newValue, true);
-        }
-        else if (value.isStringValue()) {
-            String stringValue = value.asStringValue().asString();
-            String newValue = formatTimestampString(task, path, stringValue);
-            return (Objects.equals(newValue, stringValue)) ? value : ValueFactory.newString(newValue);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(outputColumn);
         }
         else {
-            return value;
-        }
-    }
-
-    private String formatTimestampString(PluginTask task, String name, String value)
-            throws TimestampParseException
-    {
-        TimestampParser parser = timestampParserMap.get(name);
-        TimestampFormatter formatter = timestampFormatterMap.get(name);
-        if (formatter == null || parser == null) {
-            return value;
-        }
-        try {
-            Timestamp timestamp = parser.parse(value);
-            return formatter.format(timestamp);
-        }
-        catch (TimestampParseException ex) {
             if (task.getStopOnInvalidRecord()) {
-                throw Throwables.propagate(ex);
+                op.run();
             }
             else {
-                logger.warn("invalid value \"{}\":\"{}\"", name, value);
-                return value;
+                try {
+                    op.run();
+                }
+                catch (final DataException ex) {
+                    logger.warn(ex.getMessage());
+                    pageBuilder.setNull(outputColumn);
+                }
             }
         }
     }
 
-
     @Override
-    public void booleanColumn(Column column)
+    public void booleanColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
         }
         else {
-            pageBuilder.setBoolean(column, pageReader.getBoolean(column));
+            pageBuilder.setBoolean(inputColumn, pageReader.getBoolean(inputColumn));
         }
     }
 
     @Override
-    public void longColumn(Column column)
+    public void longColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
         }
         else {
-            pageBuilder.setLong(column, pageReader.getLong(column));
+            pageBuilder.setLong(inputColumn, pageReader.getLong(inputColumn));
         }
     }
 
     @Override
-    public void doubleColumn(Column column)
+    public void doubleColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
         }
         else {
-            pageBuilder.setDouble(column, pageReader.getDouble(column));
+            pageBuilder.setDouble(inputColumn, pageReader.getDouble(inputColumn));
         }
     }
 
     @Override
-    public void stringColumn(Column column)
+    public void stringColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
             return;
         }
-        String value = pageReader.getString(column);
-        String formatted = formatTimestampString(task, column.getName(), value);
-        pageBuilder.setString(column, formatted);
+        final Column outputColumn = outputColumnMap.get(inputColumn.getName());
+        PageBuildable op = new PageBuildable() {
+            public void run() throws DataException
+            {
+                columnCaster.setFromString(outputColumn, pageReader.getString(inputColumn));
+            }
+        };
+        withStopOnInvalidRecord(op, inputColumn, outputColumn);
     }
 
     @Override
-    public void jsonColumn(Column column)
+    public void timestampColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            String path = new StringBuilder("$.").append(column.getName()).toString();
-            Value value = pageReader.getJson(column);
-            Value formatted = formatTimestampStringRecursively(task, path, value);
-            pageBuilder.setJson(column, formatted);
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
+            return;
         }
+        final Column outputColumn = outputColumnMap.get(inputColumn.getName());
+        PageBuildable op = new PageBuildable() {
+            public void run() throws DataException
+            {
+                columnCaster.setFromTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
+            }
+        };
+        withStopOnInvalidRecord(op, inputColumn, outputColumn);
     }
 
     @Override
-    public void timestampColumn(Column column)
+    public void jsonColumn(final Column inputColumn)
     {
-        if (pageReader.isNull(column)) {
-            pageBuilder.setNull(column);
-        }
-        else {
-            pageBuilder.setTimestamp(column, pageReader.getTimestamp(column));
+        if (pageReader.isNull(inputColumn)) {
+            pageBuilder.setNull(inputColumn);
+            return;
         }
+        final Column outputColumn = outputColumnMap.get(inputColumn.getName());
+        columnCaster.setFromJson(outputColumn, pageReader.getJson(inputColumn));
     }
 }

data/src/main/java/org/embulk/filter/timestamp_format/JsonCaster.java

@@ -0,0 +1,54 @@
+package org.embulk.filter.timestamp_format;
+
+import org.embulk.filter.timestamp_format.cast.StringCast;
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.Type;
+import org.msgpack.value.StringValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+
+class JsonCaster
+{
+    private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
+    private final PluginTask task;
+    private final HashMap<String, TimestampParser> timestampParserMap;
+    private final HashMap<String, TimestampFormatter> timestampFormatterMap;
+
+    JsonCaster(PluginTask task,
+               HashMap<String, TimestampParser> timestampParserMap,
+               HashMap<String, TimestampFormatter> timestampFormatterMap)
+    {
+        this.task = task;
+        this.timestampParserMap = timestampParserMap;
+        this.timestampFormatterMap = timestampFormatterMap;
+    }
+
+    public Value fromString(ColumnConfig columnConfig, StringValue value)
+    {
+        Type outputType = columnConfig.getType();
+        TimestampParser parser = timestampParserMap.get(columnConfig.getName());
+        if (outputType instanceof StringType) {
+            TimestampFormatter formatter = timestampFormatterMap.get(columnConfig.getName());
+            return ValueFactory.newString(StringCast.asString(value.asString(), parser, formatter));
+        }
+        else if (outputType instanceof LongType) {
+            return ValueFactory.newInteger(StringCast.asLong(value.asString(), parser));
+        }
+        else if (outputType instanceof DoubleType) {
+            return ValueFactory.newFloat(StringCast.asDouble(value.asString(), parser));
+        }
+        else {
+            assert false;
+            return null;
+        }
+    }
+}

data/src/main/java/org/embulk/filter/timestamp_format/JsonVisitor.java

@@ -0,0 +1,119 @@
+package org.embulk.filter.timestamp_format;
+
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.ColumnConfig;
+import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask;
+
+import org.embulk.spi.Exec;
+import org.msgpack.value.ArrayValue;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+public class JsonVisitor
+{
+    private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
+    private final PluginTask task;
+    private final JsonCaster jsonCaster;
+    private final HashMap<String, ColumnConfig> jsonPathColumnConfigMap = new HashMap<>();
+    private final HashSet<String> shouldVisitSet = new HashSet<>();
+
+    JsonVisitor(PluginTask task, JsonCaster jsonCaster)
+    {
+        this.task = task;
+        this.jsonCaster = jsonCaster;
+
+        buildJsonPathColumnConfigMap();
+        buildShouldVisitSet();
+    }
+
+    private void buildJsonPathColumnConfigMap()
+    {
+        // json path => Type
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            String name = columnConfig.getName();
+            if (!name.startsWith("$.")) {
+                continue;
+            }
+            this.jsonPathColumnConfigMap.put(name, columnConfig);
+        }
+    }
+
+    private void buildShouldVisitSet()
+    {
+        // json partial path => Boolean to avoid unnecessary type: json visit
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            String name = columnConfig.getName();
+            if (!name.startsWith("$.")) {
+                continue;
+            }
+            String[] parts = name.split("\\.");
+            StringBuilder partialPath = new StringBuilder("$");
+            for (int i = 1; i < parts.length; i++) {
+                if (parts[i].contains("[")) {
+                    String[] arrayParts = parts[i].split("\\[");
+                    partialPath.append(".").append(arrayParts[0]);
+                    this.shouldVisitSet.add(partialPath.toString());
+                    for (int j = 1; j < arrayParts.length; j++) {
+                        partialPath.append("[").append(arrayParts[j]);
+                        this.shouldVisitSet.add(partialPath.toString());
+                    }
+                }
+                else {
+                    partialPath.append(".").append(parts[i]);
+                    this.shouldVisitSet.add(partialPath.toString());
+                }
+            }
+        }
+    }
+
+    private boolean shouldVisit(String jsonPath)
+    {
+        return shouldVisitSet.contains(jsonPath);
+    }
+
+    public Value visit(String jsonPath, Value value)
+    {
+        if (!shouldVisit(jsonPath)) {
+            return value;
+        }
+        if (value.isArrayValue()) {
+            ArrayValue arrayValue = value.asArrayValue();
+            int size = arrayValue.size();
+            Value[] newValue = new Value[size];
+            for (int i = 0; i < size; i++) {
+                String k = new StringBuilder(jsonPath).append("[").append(Integer.toString(i)).append("]").toString();
+                Value v = arrayValue.get(i);
+                newValue[i] = visit(k, v);
+            }
+            return ValueFactory.newArray(newValue, true);
+        }
+        else if (value.isMapValue()) {
+            MapValue mapValue = value.asMapValue();
+            int size = mapValue.size() * 2;
+            Value[] newValue = new Value[size];
+            int i = 0;
+            for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
+                Value k = entry.getKey();
+                Value v = entry.getValue();
+                String newPath = new StringBuilder(jsonPath).append(".").append(k.asStringValue().asString()).toString();
+                Value r = visit(newPath, v);
+                newValue[i++] = k;
+                newValue[i++] = r;
+            }
+            return ValueFactory.newMap(newValue, true);
+        }
+        else if (value.isStringValue()) {
+            ColumnConfig columnConfig = jsonPathColumnConfigMap.get(jsonPath);
+            return jsonCaster.fromString(columnConfig, value.asStringValue());
+        }
+        else {
+            return value;
+        }
+    }
+}

data/src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java

@@ -1,12 +1,15 @@
 package org.embulk.filter.timestamp_format;
 
+import com.google.common.collect.ImmutableList;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
+import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Page;
@@ -15,6 +18,11 @@ import org.embulk.spi.PageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
 
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
 import org.jruby.embed.ScriptingContainer;
 import org.slf4j.Logger;
 
@@ -24,20 +32,22 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
 {
     private static final Logger logger = Exec.getLogger(TimestampFormatFilterPlugin.class);
 
-    public TimestampFormatFilterPlugin()
-    {
-    }
+    public TimestampFormatFilterPlugin() {}
 
     // NOTE: This is not spi.ColumnConfig
-    public interface ColumnConfig extends Task,
+    interface ColumnConfig extends Task,
             TimestampParser.TimestampColumnOption, TimestampFormatter.TimestampColumnOption
     {
         @Config("name")
         String getName();
+
+        @Config("type")
+        @ConfigDefault("\"string\"")
+        Type getType();
     }
 
-    public interface PluginTask extends Task,
-           TimestampParser.Task, TimestampFormatter.Task
+    interface PluginTask extends Task,
+            TimestampParser.Task, TimestampFormatter.Task
     {
         @Config("columns")
         @ConfigDefault("[]")
@@ -57,12 +67,20 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        configure(task, inputSchema);
+        Schema outputSchema = buildOuputSchema(task, inputSchema);
+        control.run(task.dump(), outputSchema);
+    }
+
+    private void configure(PluginTask task, Schema inputSchema)
+    {
         List<ColumnConfig> columns = task.getColumns();
+
         // throw if column does not exist
         for (ColumnConfig columnConfig : columns) {
             String name = columnConfig.getName();
             if (name.startsWith("$.")) {
-                String firstName = name.split("\\.", 3)[1];
+                String firstName = name.split("\\.", 3)[1]; // check only top level column name
                 inputSchema.lookupColumn(firstName);
             }
             else {
@@ -70,7 +88,55 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
             }
         }
 
-        control.run(task.dump(), inputSchema);
+        // throw if column type is not string or timestamp
+        for (ColumnConfig columnConfig : columns) {
+            Type type = columnConfig.getType();
+            boolean acceptable = false;
+            if (type instanceof StringType) {
+                continue;
+            }
+            else if (type instanceof TimestampType) {
+                continue;
+            }
+            else if (type instanceof LongType) {
+                continue;
+            }
+            else if (type instanceof DoubleType) {
+                continue;
+            }
+            else {
+                throw new ConfigException("column type must be string, timestamp, long, or double");
+            }
+        }
+    }
+
+    private Schema buildOuputSchema(final PluginTask task, final Schema inputSchema)
+    {
+        List<ColumnConfig> columnConfigs = task.getColumns();
+        ImmutableList.Builder<Column> builder = ImmutableList.builder();
+        int i = 0;
+        for (Column inputColumn : inputSchema.getColumns()) {
+            String name = inputColumn.getName();
+            Type   type = inputColumn.getType();
+            ColumnConfig columnConfig = getColumnConfig(name, columnConfigs);
+            if (columnConfig != null) {
+                type = columnConfig.getType();
+            }
+            Column outputColumn = new Column(i++, name, type);
+            builder.add(outputColumn);
+        }
+        return new Schema(builder.build());
+    }
+
+    private ColumnConfig getColumnConfig(String name, List<ColumnConfig> columnConfigs)
+    {
+        // hash should be faster, though
+        for (ColumnConfig columnConfig : columnConfigs) {
+            if (columnConfig.getName().equals(name)) {
+                return columnConfig;
+            }
+        }
+        return null;
     }
 
     @Override
@@ -82,7 +148,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
         return new PageOutput() {
             private PageReader pageReader = new PageReader(inputSchema);
             private PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
-            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, pageReader, pageBuilder);
+            private ColumnVisitorImpl visitor = new ColumnVisitorImpl(task, inputSchema, outputSchema, pageReader, pageBuilder);
 
             @Override
             public void finish()
@@ -102,7 +168,7 @@ public class TimestampFormatFilterPlugin implements FilterPlugin
                 pageReader.setPage(page);
 
                 while (pageReader.nextRecord()) {
-                    outputSchema.visitColumns(visitor);
+                    inputSchema.visitColumns(visitor);
                     pageBuilder.addRecord();
                 }
             }

data/src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java

@@ -10,10 +10,10 @@ import org.embulk.filter.timestamp_format.TimestampFormatFilterPlugin.PluginTask
 import org.embulk.spi.time.JRubyTimeParserHelper;
 import org.embulk.spi.time.JRubyTimeParserHelperFactory;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParseException;
 
 import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
 
+import org.embulk.spi.time.TimestampParseException;
 import org.joda.time.DateTimeZone;
 import org.jruby.embed.ScriptingContainer;
 

data/src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java

@@ -0,0 +1,59 @@
+package org.embulk.filter.timestamp_format.cast;
+
+import org.embulk.filter.timestamp_format.TimestampFormatter;
+import org.embulk.filter.timestamp_format.TimestampParser;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampParseException;
+
+public class StringCast
+{
+    private StringCast() {}
+
+    private static String buildErrorMessage(String value)
+    {
+        return String.format("failed to parse string: \"%s\"", value);
+    }
+
+    public static String asString(String value, TimestampParser parser, TimestampFormatter formatter) throws DataException
+    {
+        try {
+            Timestamp timestamp = parser.parse(value);
+            return formatter.format(timestamp);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage(value), ex);
+        }
+    }
+
+    public static Timestamp asTimestamp(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            return parser.parse(value);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage(value), ex);
+        }
+    }
+
+    public static long asLong(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            Timestamp timestamp = parser.parse(value);
+            return timestamp.getEpochSecond();
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage(value), ex);
+        }
+    }
+    public static double asDouble(String value, TimestampParser parser) throws DataException
+    {
+        try {
+            Timestamp timestamp = parser.parse(value);
+            return TimestampCast.asDouble(timestamp);
+        }
+        catch (TimestampParseException ex) {
+            throw new DataException(buildErrorMessage(value), ex);
+        }
+    }
+}

data/src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java

@@ -0,0 +1,32 @@
+package org.embulk.filter.timestamp_format.cast;
+
+import org.embulk.filter.timestamp_format.TimestampFormatter;
+import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
+
+public class TimestampCast
+{
+    private TimestampCast() {}
+
+    public static String asString(Timestamp value, TimestampFormatter formatter) throws DataException
+    {
+        return formatter.format(value);
+    }
+
+    public static Timestamp asTimestamp(Timestamp value) throws DataException
+    {
+        return value;
+    }
+
+    public static long asLong(Timestamp value) throws DataException
+    {
+        return value.getEpochSecond();
+    }
+
+    public static double asDouble(Timestamp value) throws DataException
+    {
+        long epoch = value.getEpochSecond();
+        int nano = value.getNano();
+        return (double) epoch + ((double) nano / 1000000000.0);
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-timestamp_format
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-26 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,20 +38,6 @@ dependencies:
         version: '10.0'
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: embulk-parser-jsonl
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  prerelease: false
-  type: :development
 description: A filter plugin for Embulk to change timestamp format.
 email:
 - sonots@gmail.com
@@ -66,19 +52,27 @@ files:
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
-- example/example.jsonl
 - example/example.yml
+- example/json_example.jsonl
+- example/json_example.yml
+- example/string_example.yml
+- example/timestamp_example.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/timestamp_format.rb
+- src/main/java/org/embulk/filter/timestamp_format/ColumnCaster.java
 - src/main/java/org/embulk/filter/timestamp_format/ColumnVisitorImpl.java
+- src/main/java/org/embulk/filter/timestamp_format/JsonCaster.java
+- src/main/java/org/embulk/filter/timestamp_format/JsonVisitor.java
 - src/main/java/org/embulk/filter/timestamp_format/TimestampFormatFilterPlugin.java
 - src/main/java/org/embulk/filter/timestamp_format/TimestampFormatter.java
 - src/main/java/org/embulk/filter/timestamp_format/TimestampParser.java
+- src/main/java/org/embulk/filter/timestamp_format/cast/StringCast.java
+- src/main/java/org/embulk/filter/timestamp_format/cast/TimestampCast.java
 - src/test/java/org/embulk/filter/TestTimestampFormatFilterPlugin.java
-- classpath/embulk-filter-timestamp_format-0.1.4.jar
+- classpath/embulk-filter-timestamp_format-0.1.5.jar
 homepage: https://github.com/sonots/embulk-filter-timestamp_format
 licenses:
 - MIT

data/example/example.jsonl

@@ -1,2 +0,0 @@
-{"timestamp":"2015-07-12 15:00:00 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00 UTC"}}
-{"timestamp":"2015-07-12 15:00:00.1 UTC","record":{"record":[{"timestamp":"2015-07-12 15:00:00.1 UTC"}]},"ignore_record":{"timestamp":"2015-07-12 15:00:00.1 UTC"}}