embulk-filter-typecast 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e8a2482d0bd6fc6109bb8763f9a1f9db6b0733db
-  data.tar.gz: b80c4d01e823e3a4b59594002277fc78e1ee0ef9
+  metadata.gz: 0ced2804053187863055f59aa064418cd0b8d7aa
+  data.tar.gz: 447a4328482d178a479b404bfb7af62c61a2c1f3
 SHA512:
-  metadata.gz: 7473b57b158d8936e14015358dcd21d3bfe852f0b7c33de10da4ad38aa4553b3ddf39e5b41c9ed11df32713c42a33506aba22820b2236590ab124035a1056783
-  data.tar.gz: 354bf8ab52c2ee5ba124af046abea9e311213c4ebaa03991f432504cc87094a0e1d4815c3c94434ee843de836f6e2146331540fd23b286071c000011bb64c55a
+  metadata.gz: 2833a31678457b155864532d95de1af40594ffa7d5ffee4d489bcc4d993b444dacd18adbbcac444b2ad6b2fe85b2fbb837fa559abbdadd13010a98009894ec1d
+  data.tar.gz: f668410813b7109c96a6277b6ecd0b576334f74261f2c5ce5d941ddc0b6b6e2b4e84d4daeedde49b3a86abc140cc99481fa7a5a0e0b330a9494efccd0cd5eaaf

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.3 (2016-05-19)
+
+Enhancements:
+
+* Support JSONPath array wildcard
+
 # 0.1.2 (2016-04-28)
 
 Enhancements:

data/README.md

@@ -19,6 +19,18 @@ A filter plugin for Embulk to cast column type.
 
 See [example.csv](./example/example.csv) and [example.yml](./example/example.yml).
 
+## JSONPath (like) name
+
+For `type: json` column, you can specify [JSONPath](http://goessner.net/articles/JsonPath/) for column's name as:
+
+```
+$.payload.key1
+$.payload.array[0]
+$.payload.array[*]
+```
+
+NOTE: JSONPath syntax is not fully supported
+
 ## ToDo
 
 * Write test

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.2"
+version = "0.1.3"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/empty.yml

@@ -0,0 +1,24 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: 'NULL'
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
+      - {name: "null",    type: string}
+      - {name: long,      type: long}
+      - {name: string,    type: string}
+      - {name: double,    type: double}
+      - {name: json1,     type: json}
+      - {name: json2,     type: json}
+      - {name: ignore,    type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
+      - {name: boolean,   type: boolean}
+filters:
+  - type: typecast
+out:
+  type: "null"

data/example/example.csv

@@ -1,11 +1,11 @@
-timestamp,null,long,string,double,json1,json2,boolean
-2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"0"},{"long":0},true
-2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"1"},{"long":1},true
-2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,{"string":"2"},{"long":2},true
-2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,{"string":"3"},{"long":3},true
-2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,{"string":"4"},{"long":4},true
-2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,{"string":"5"},{"long":5},false
-2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,{"string":"6"},{"long":6},false
-2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,{"string":"7"},{"long":7},false
-2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"8"},{"long":8},false
-2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"9"},{"long":9},false
+timestamp,null,long,string,double,json1,json2,array_str,array_int,ignore,boolean
+2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"0"},{"long":0},["0"],[0],2015-07-13,true
+2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"1"},{"long":1},["1"],[1],2015-07-13,true
+2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,{"string":"2"},{"long":2},["2"],[2],2015-07-13,true
+2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,{"string":"3"},{"long":3},["3"],[3],2015-07-13,true
+2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,{"string":"4"},{"long":4},["4"],[4],2015-07-13,true
+2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,{"string":"5"},{"long":5},["5"],[5],2015-07-13,false
+2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,{"string":"6"},{"long":6},["6"],[6],2015-07-13,false
+2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,{"string":"7"},{"long":7},["7"],[7],2015-07-13,false
+2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"8"},{"long":8},["8"],[8],2015-07-13,false
+2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"9"},{"long":9},["9"],[9],2015-07-13,false

data/example/example.yml

@@ -16,6 +16,9 @@ in:
       - {name: double,    type: double}
       - {name: json1,     type: json}
       - {name: json2,     type: json}
+      - {name: array_str, type: json}
+      - {name: array_int, type: json}
+      - {name: ignore,    type: timestamp, format: "%Y-%m-%d", timezone: "+09:00"}
       - {name: boolean,   type: boolean}
 filters:
   - type: typecast
@@ -29,5 +32,7 @@ filters:
       - {name: boolean,   type: string}
       - {name: "$.json1.string", type: string}
       - {name: "$.json2.long",   type: string}
+      - {name: "$.array_str[0]", type: long}
+      - {name: "$.array_int[*]", type: string}
 out:
   type: "null"

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = 'embulk-filter-typecast'

data/src/main/java/org/embulk/filter/typecast/ColumnCaster.java

@@ -1,11 +1,15 @@
 package org.embulk.filter.typecast;
 
-import org.embulk.filter.typecast.cast.*;
-
+import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
 import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
 
+import org.embulk.filter.typecast.cast.BooleanCast;
+import org.embulk.filter.typecast.cast.DoubleCast;
+import org.embulk.filter.typecast.cast.JsonCast;
+import org.embulk.filter.typecast.cast.LongCast;
+import org.embulk.filter.typecast.cast.StringCast;
+import org.embulk.filter.typecast.cast.TimestampCast;
 import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
 import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageReader;
@@ -13,11 +17,19 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.type.*;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.JsonType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.type.Type;
+import org.joda.time.DateTimeZone;
 import org.msgpack.value.Value;
 
 import org.slf4j.Logger;
 
+import java.util.HashMap;
 
 class ColumnCaster
 {
@@ -27,6 +39,8 @@ class ColumnCaster
     private final Schema outputSchema;
     private final PageReader pageReader;
     private final PageBuilder pageBuilder;
+    private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<>();
+    private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<>();
     private final JsonVisitor jsonVisitor;
 
     ColumnCaster(TypecastFilterPlugin.PluginTask task, Schema inputSchema, Schema outputSchema,
@@ -37,24 +51,78 @@ class ColumnCaster
         this.outputSchema = outputSchema;
         this.pageReader = pageReader;
         this.pageBuilder = pageBuilder;
+
+        buildTimestampParserMap();
+        buildTimestampFormatterMap();
         this.jsonVisitor = new JsonVisitor(task, inputSchema, outputSchema);
     }
 
-    public void setFromBoolean(Column outputColumn, boolean value) {
+    private void buildTimestampParserMap()
+    {
+        // columnName => TimestampParser
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            if (columnConfig.getName().startsWith("$.")) {
+                continue; // type: json columns do not support type: timestamp
+            }
+            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
+            if (inputColumn.getType() instanceof StringType && columnConfig.getType() instanceof TimestampType) {
+                TimestampParser parser = getTimestampParser(columnConfig, task);
+                this.timestampParserMap.put(columnConfig.getName(), parser);
+            }
+        }
+    }
+
+    private void buildTimestampFormatterMap()
+    {
+        // columnName => TimestampFormatter
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            if (columnConfig.getName().startsWith("$.")) {
+                continue; // type: json columns do not have type: timestamp
+            }
+            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
+            if (inputColumn.getType() instanceof TimestampType && columnConfig.getType() instanceof StringType) {
+                TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
+                this.timestampFormatterMap.put(columnConfig.getName(), parser);
+            }
+        }
+    }
+
+    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
+    {
+        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
+        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
+        return new TimestampParser(task.getJRuby(), format, timezone);
+    }
+
+    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
+    {
+        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
+        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
+        return new TimestampFormatter(task.getJRuby(), format, timezone);
+    }
+
+    public void setFromBoolean(Column outputColumn, boolean value)
+    {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, BooleanCast.asBoolean(value));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             pageBuilder.setLong(outputColumn, BooleanCast.asLong(value));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             pageBuilder.setDouble(outputColumn, BooleanCast.asDouble(value));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             pageBuilder.setString(outputColumn, BooleanCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
+        }
+        else if (outputType instanceof TimestampType) {
             pageBuilder.setTimestamp(outputColumn, BooleanCast.asTimestamp(value));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             pageBuilder.setJson(outputColumn, BooleanCast.asJson(value));
-        } else {
+        }
+        else {
             assert (false);
         }
     }
@@ -64,86 +132,107 @@ class ColumnCaster
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, LongCast.asBoolean(value));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             pageBuilder.setLong(outputColumn, LongCast.asLong(value));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             pageBuilder.setDouble(outputColumn, LongCast.asDouble(value));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             pageBuilder.setString(outputColumn, LongCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
+        }
+        else if (outputType instanceof TimestampType) {
             pageBuilder.setTimestamp(outputColumn, LongCast.asTimestamp(value));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             pageBuilder.setJson(outputColumn, LongCast.asJson(value));
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
         }
     }
 
     public void setFromDouble(Column outputColumn, double value)
     {
-        try {
-            Type outputType = outputColumn.getType();
-            if (outputType instanceof BooleanType) {
-                pageBuilder.setBoolean(outputColumn, DoubleCast.asBoolean(value));
-            } else if (outputType instanceof LongType) {
-                pageBuilder.setLong(outputColumn, DoubleCast.asLong(value));
-            } else if (outputType instanceof DoubleType) {
-                pageBuilder.setDouble(outputColumn, DoubleCast.asDouble(value));
-            } else if (outputType instanceof StringType) {
-                pageBuilder.setString(outputColumn, DoubleCast.asString(value));
-            } else if (outputType instanceof TimestampType) {
-                pageBuilder.setTimestamp(outputColumn, DoubleCast.asTimestamp(value));
-            } else if (outputType instanceof JsonType) {
-                pageBuilder.setJson(outputColumn, DoubleCast.asJson(value));
-            } else {
-                assert (false);
-            }
+        Type outputType = outputColumn.getType();
+        if (outputType instanceof BooleanType) {
+            pageBuilder.setBoolean(outputColumn, DoubleCast.asBoolean(value));
         }
-        catch (DataException ex) {
-
+        else if (outputType instanceof LongType) {
+            pageBuilder.setLong(outputColumn, DoubleCast.asLong(value));
+        }
+        else if (outputType instanceof DoubleType) {
+            pageBuilder.setDouble(outputColumn, DoubleCast.asDouble(value));
+        }
+        else if (outputType instanceof StringType) {
+            pageBuilder.setString(outputColumn, DoubleCast.asString(value));
+        }
+        else if (outputType instanceof TimestampType) {
+            pageBuilder.setTimestamp(outputColumn, DoubleCast.asTimestamp(value));
+        }
+        else if (outputType instanceof JsonType) {
+            pageBuilder.setJson(outputColumn, DoubleCast.asJson(value));
+        }
+        else {
+            assert false;
         }
     }
 
-    public void setFromString(Column outputColumn, String value, TimestampParser timestampParser)
+    public void setFromString(Column outputColumn, String value)
     {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, StringCast.asBoolean(value));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             pageBuilder.setLong(outputColumn, StringCast.asLong(value));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             pageBuilder.setDouble(outputColumn, StringCast.asDouble(value));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             pageBuilder.setString(outputColumn, StringCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
+        }
+        else if (outputType instanceof TimestampType) {
+            TimestampParser timestampParser = timestampParserMap.get(outputColumn.getName());
             pageBuilder.setTimestamp(outputColumn, StringCast.asTimestamp(value, timestampParser));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             Value jsonValue = StringCast.asJson(value);
             String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
             Value castedValue = jsonVisitor.visit(jsonPath, jsonValue);
             pageBuilder.setJson(outputColumn, castedValue);
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
         }
     }
 
-    public void setFromTimestamp(Column outputColumn, Timestamp value, TimestampFormatter timestampFormatter)
+    public void setFromTimestamp(Column outputColumn, Timestamp value)
     {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, TimestampCast.asBoolean(value));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             pageBuilder.setLong(outputColumn, TimestampCast.asLong(value));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             pageBuilder.setDouble(outputColumn, TimestampCast.asDouble(value));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
+            TimestampFormatter timestampFormatter = timestampFormatterMap.get(outputColumn.getName());
             pageBuilder.setString(outputColumn, TimestampCast.asString(value, timestampFormatter));
-        } else if (outputType instanceof TimestampType) {
+        }
+        else if (outputType instanceof TimestampType) {
             pageBuilder.setTimestamp(outputColumn, TimestampCast.asTimestamp(value));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             pageBuilder.setJson(outputColumn, TimestampCast.asJson(value));
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
         }
     }
 
@@ -154,18 +243,24 @@ class ColumnCaster
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, JsonCast.asBoolean(castedValue));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             pageBuilder.setLong(outputColumn, JsonCast.asLong(castedValue));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             pageBuilder.setDouble(outputColumn, JsonCast.asDouble(castedValue));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             pageBuilder.setString(outputColumn, JsonCast.asString(castedValue));
-        } else if (outputType instanceof TimestampType) {
+        }
+        else if (outputType instanceof TimestampType) {
             pageBuilder.setTimestamp(outputColumn, JsonCast.asTimestamp(castedValue));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             pageBuilder.setJson(outputColumn, JsonCast.asJson(castedValue));
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
         }
     }
 }

data/src/main/java/org/embulk/filter/typecast/ColumnVisitorImpl.java

@@ -1,21 +1,19 @@
 package org.embulk.filter.typecast;
 
-import org.embulk.spi.*;
-import org.embulk.spi.type.StringType;
-import org.embulk.spi.type.TimestampType;
-
-import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
 import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
 
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-import org.joda.time.DateTimeZone;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
 import org.slf4j.Logger;
 
 import java.util.HashMap;
 
-public class ColumnVisitorImpl
-        implements ColumnVisitor
+class ColumnVisitorImpl implements ColumnVisitor
 {
     private static final Logger logger = Exec.getLogger(TypecastFilterPlugin.class);
     private final PluginTask task;
@@ -24,8 +22,6 @@ public class ColumnVisitorImpl
     private final PageReader pageReader;
     private final PageBuilder pageBuilder;
     private final HashMap<String, Column> outputColumnMap = new HashMap<>();
-    private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<>();
-    private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<>();
     private final ColumnCaster columnCaster;
 
     ColumnVisitorImpl(PluginTask task, Schema inputSchema, Schema outputSchema,
@@ -40,8 +36,6 @@ public class ColumnVisitorImpl
         this.columnCaster = new ColumnCaster(task, inputSchema, outputSchema, pageReader, pageBuilder);
 
         buildOutputColumnMap();
-        buildTimestampParserMap();
-        buildTimestampFormatterMap();
     }
 
     private void buildOutputColumnMap()
@@ -52,66 +46,26 @@ public class ColumnVisitorImpl
         }
     }
 
-    private void buildTimestampParserMap()
-    {
-        // columnName => TimestampParser
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            if (columnConfig.getName().startsWith("$.")) {
-                continue; // type: json columns do not support type: timestamp
-            }
-            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
-            if (inputColumn.getType() instanceof StringType && columnConfig.getType() instanceof TimestampType) {
-                TimestampParser parser = getTimestampParser(columnConfig, task);
-                this.timestampParserMap.put(columnConfig.getName(), parser);
-            }
-        }
-    }
-
-    private void buildTimestampFormatterMap()
-    {
-        // columnName => TimestampFormatter
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            if (columnConfig.getName().startsWith("$.")) {
-                continue; // type: json columns do not have type: timestamp
-            }
-            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
-            if (inputColumn.getType() instanceof TimestampType && columnConfig.getType() instanceof StringType) {
-                TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
-                this.timestampFormatterMap.put(columnConfig.getName(), parser);
-            }
-        }
-    }
-
-    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
-    {
-        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
-        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
-        return new TimestampParser(task.getJRuby(), format, timezone);
-    }
-
-    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
-    {
-        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
-        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
-        return new TimestampFormatter(task.getJRuby(), format, timezone);
-    }
-
     private interface PageBuildable
     {
-        public void run() throws DataException;
+        void run() throws DataException;
     }
 
-    private void withStopOnInvalidRecord(final PageBuildable op, final Column inputColumn, final Column outputColumn) throws DataException {
+    private void withStopOnInvalidRecord(final PageBuildable op, final Column inputColumn, final Column outputColumn)
+            throws DataException
+    {
         if (pageReader.isNull(inputColumn)) {
             pageBuilder.setNull(outputColumn);
         }
         else {
             if (task.getStopOnInvalidRecord()) {
                 op.run();
-            } else {
+            }
+            else {
                 try {
                     op.run();
-                } catch (final DataException ex) {
+                }
+                catch (final DataException ex) {
                     logger.warn(ex.getMessage());
                     pageBuilder.setNull(outputColumn);
                 }
@@ -124,7 +78,8 @@ public class ColumnVisitorImpl
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
+            public void run() throws DataException
+            {
                 columnCaster.setFromBoolean(outputColumn, pageReader.getBoolean(inputColumn));
             }
         };
@@ -136,7 +91,8 @@ public class ColumnVisitorImpl
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
+            public void run() throws DataException
+            {
                 columnCaster.setFromLong(outputColumn, pageReader.getLong(inputColumn));
             }
         };
@@ -148,7 +104,8 @@ public class ColumnVisitorImpl
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
+            public void run() throws DataException
+            {
                 columnCaster.setFromDouble(outputColumn, pageReader.getDouble(inputColumn));
             }
         };
@@ -159,10 +116,10 @@ public class ColumnVisitorImpl
     public void stringColumn(final Column inputColumn)
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
-        final TimestampParser timestampParser = timestampParserMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
-                columnCaster.setFromString(outputColumn, pageReader.getString(inputColumn), timestampParser);
+            public void run() throws DataException
+            {
+                columnCaster.setFromString(outputColumn, pageReader.getString(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -172,10 +129,10 @@ public class ColumnVisitorImpl
     public void timestampColumn(final Column inputColumn)
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
-        final TimestampFormatter timestampFormatter = timestampFormatterMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
-                columnCaster.setFromTimestamp(outputColumn, pageReader.getTimestamp(inputColumn), timestampFormatter);
+            public void run() throws DataException
+            {
+                columnCaster.setFromTimestamp(outputColumn, pageReader.getTimestamp(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -186,7 +143,8 @@ public class ColumnVisitorImpl
     {
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
-            public void run() throws DataException {
+            public void run() throws DataException
+            {
                 columnCaster.setFromJson(outputColumn, pageReader.getJson(inputColumn));
             }
         };

data/src/main/java/org/embulk/filter/typecast/JsonCaster.java

@@ -1,34 +1,45 @@
 package org.embulk.filter.typecast;
 
-import org.embulk.filter.typecast.cast.*;
-
+import org.embulk.filter.typecast.cast.BooleanCast;
+import org.embulk.filter.typecast.cast.DoubleCast;
+import org.embulk.filter.typecast.cast.LongCast;
+import org.embulk.filter.typecast.cast.StringCast;
 import org.embulk.spi.DataException;
-import org.embulk.spi.type.*;
+import org.embulk.spi.type.BooleanType;
+import org.embulk.spi.type.DoubleType;
+import org.embulk.spi.type.JsonType;
+import org.embulk.spi.type.LongType;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.Type;
 import org.msgpack.value.BooleanValue;
-import org.msgpack.value.IntegerValue;
 import org.msgpack.value.FloatValue;
+import org.msgpack.value.IntegerValue;
 import org.msgpack.value.StringValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
 class JsonCaster
 {
-    public JsonCaster()
-    {
-    }
+    public JsonCaster() {}
 
-    public Value fromBoolean(Type outputType, BooleanValue value) {
+    public Value fromBoolean(Type outputType, BooleanValue value)
+    {
         if (outputType instanceof BooleanType) {
             return value;
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             return ValueFactory.newInteger(BooleanCast.asLong(value.getBoolean()));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             return ValueFactory.newFloat(BooleanCast.asDouble(value.getBoolean()));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             return ValueFactory.newString(BooleanCast.asString(value.getBoolean()));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast boolean to json: \"%s\"", value));
-        } else {
+        }
+        else {
             assert (false);
             return null;
         }
@@ -38,16 +49,21 @@ class JsonCaster
     {
         if (outputType instanceof BooleanType) {
             return ValueFactory.newBoolean(LongCast.asBoolean(value.asLong()));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             return value;
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             return ValueFactory.newFloat(LongCast.asDouble(value.asLong()));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             return ValueFactory.newString(LongCast.asString(value.asLong()));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast long to json:: \"%s\"", value));
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
             return null;
         }
     }
@@ -56,15 +72,20 @@ class JsonCaster
     {
         if (outputType instanceof BooleanType) {
             return ValueFactory.newBoolean(DoubleCast.asBoolean(value.toDouble()));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             return ValueFactory.newInteger(DoubleCast.asLong(value.toDouble()));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             return value;
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             return ValueFactory.newString(DoubleCast.asString(value.toDouble()));
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast double to json:: \"%s\"", value));
-        } else {
+        }
+        else {
             assert (false);
             return null;
         }
@@ -74,16 +95,21 @@ class JsonCaster
     {
         if (outputType instanceof BooleanType) {
             return ValueFactory.newBoolean(StringCast.asBoolean(value.asString()));
-        } else if (outputType instanceof LongType) {
+        }
+        else if (outputType instanceof LongType) {
             return ValueFactory.newInteger(StringCast.asLong(value.asString()));
-        } else if (outputType instanceof DoubleType) {
+        }
+        else if (outputType instanceof DoubleType) {
             return ValueFactory.newFloat(StringCast.asDouble(value.asString()));
-        } else if (outputType instanceof StringType) {
+        }
+        else if (outputType instanceof StringType) {
             return value;
-        } else if (outputType instanceof JsonType) {
+        }
+        else if (outputType instanceof JsonType) {
             return StringCast.asJson(value.asString());
-        } else {
-            assert(false);
+        }
+        else {
+            assert false;
             return null;
         }
     }

data/src/main/java/org/embulk/filter/typecast/JsonVisitor.java

@@ -1,15 +1,16 @@
 package org.embulk.filter.typecast;
 
-import org.embulk.spi.*;
+import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
+import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
+
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
 import org.embulk.spi.type.Type;
 import org.msgpack.value.ArrayValue;
 import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
-import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
-import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
-
 import org.slf4j.Logger;
 
 import java.util.HashMap;
@@ -65,6 +66,7 @@ public class JsonVisitor
                     partialPath.append(".").append(arrayParts[0]);
                     this.shouldVisitSet.add(partialPath.toString());
                     for (int j = 1; j < arrayParts.length; j++) {
+                        // Supports both [0] and [*]
                         partialPath.append("[").append(arrayParts[j]);
                         this.shouldVisitSet.add(partialPath.toString());
                     }
@@ -93,6 +95,9 @@ public class JsonVisitor
             Value[] newValue = new Value[size];
             for (int i = 0; i < size; i++) {
                 String k = new StringBuilder(jsonPath).append("[").append(Integer.toString(i)).append("]").toString();
+                if (!shouldVisit(k)) {
+                    k = new StringBuilder(jsonPath).append("[*]").toString(); // try [*] too
+                }
                 Value v = arrayValue.get(i);
                 newValue[i] = visit(k, v);
             }

data/src/main/java/org/embulk/filter/typecast/TypecastFilterPlugin.java

@@ -10,11 +10,14 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
-import org.embulk.spi.*;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
 import org.joda.time.DateTimeZone;
@@ -59,7 +62,6 @@ public class TypecastFilterPlugin implements FilterPlugin
         @ConfigDefault("false")
         Boolean getStopOnInvalidRecord();
 
-
         @Config("default_timezone")
         @ConfigDefault("\"UTC\"")
         public DateTimeZone getDefaultTimeZone();
@@ -91,7 +93,8 @@ public class TypecastFilterPlugin implements FilterPlugin
             String name = columnConfig.getName();
             if (name.startsWith("$.")) { // check only top level column name
                 String firstName = name.split("\\.", 3)[1];
-                inputSchema.lookupColumn(firstName);
+                String firstNameWithoutArray = firstName.split("\\[")[0];
+                inputSchema.lookupColumn(firstNameWithoutArray);
             }
             else {
                 inputSchema.lookupColumn(name);
@@ -137,7 +140,7 @@ public class TypecastFilterPlugin implements FilterPlugin
 
     @Override
     public PageOutput open(final TaskSource taskSource, final Schema inputSchema,
-            final Schema outputSchema, final PageOutput output)
+                           final Schema outputSchema, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
 

data/src/main/java/org/embulk/filter/typecast/cast/BooleanCast.java

@@ -1,13 +1,16 @@
 package org.embulk.filter.typecast.cast;
 
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
 import org.msgpack.value.Value;
 
-public class BooleanCast {
+public class BooleanCast
+{
+    private BooleanCast() {}
+
     private static String buildErrorMessage(String as, boolean value)
     {
-        return String.format("cannot cast double to %s: \"%s\"", as, value);
+        return String.format("cannot cast boolean to %s: \"%s\"", as, value);
     }
 
     public static boolean asBoolean(boolean value) throws DataException

data/src/main/java/org/embulk/filter/typecast/cast/DoubleCast.java

@@ -1,10 +1,13 @@
 package org.embulk.filter.typecast.cast;
 
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
 import org.msgpack.value.Value;
 
-public class DoubleCast {
+public class DoubleCast
+{
+    private DoubleCast() {}
+
     private static String buildErrorMessage(String as, double value)
     {
         return String.format("cannot cast double to %s: \"%s\"", as, value);
@@ -17,7 +20,7 @@ public class DoubleCast {
 
     public static long asLong(double value) throws DataException
     {
-        return (long)value;
+        return (long) value;
     }
 
     public static double asDouble(double value) throws DataException
@@ -37,7 +40,7 @@ public class DoubleCast {
 
     public static Timestamp asTimestamp(double value) throws DataException
     {
-        long epochSecond = (long)value;
+        long epochSecond = (long) value;
         long nanoAdjustMent = (long) ((value - epochSecond) * 1000000000);
         return Timestamp.ofEpochSecond(epochSecond, nanoAdjustMent);
     }

data/src/main/java/org/embulk/filter/typecast/cast/JsonCast.java

@@ -4,10 +4,13 @@ import org.embulk.spi.DataException;
 import org.embulk.spi.time.Timestamp;
 import org.msgpack.value.Value;
 
-public class JsonCast {
+public class JsonCast
+{
+    private JsonCast() {}
+
     private static String buildErrorMessage(String as, Value value)
     {
-        return String.format("cannot cast String to %s: \"%s\"", as, value);
+        return String.format("cannot cast Json to %s: \"%s\"", as, value);
     }
 
     public static boolean asBoolean(Value value) throws DataException

data/src/main/java/org/embulk/filter/typecast/cast/LongCast.java

@@ -1,10 +1,13 @@
 package org.embulk.filter.typecast.cast;
 
-import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.DataException;
+import org.embulk.spi.time.Timestamp;
 import org.msgpack.value.Value;
 
-public class LongCast {
+public class LongCast
+{
+    private LongCast() {}
+
     private static String buildErrorMessage(String as, long value)
     {
         return String.format("cannot cast long to %s: \"%s\"", as, value);
@@ -30,7 +33,7 @@ public class LongCast {
 
     public static double asDouble(long value) throws DataException
     {
-        return (double)value;
+        return (double) value;
     }
 
     public static String asString(long value) throws DataException

data/src/main/java/org/embulk/filter/typecast/cast/StringCast.java

@@ -2,14 +2,15 @@ package org.embulk.filter.typecast.cast;
 
 import com.google.common.collect.ImmutableSet;
 import org.embulk.spi.DataException;
-import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
 import org.msgpack.value.Value;
 
-public class StringCast {
+public class StringCast
+{
     private static final JsonParser jsonParser = new JsonParser();
 
     // copy from csv plugin
@@ -29,6 +30,7 @@ public class StringCast {
                     "off", "Off", "OFF",
                     "0");
 
+    private StringCast() {}
 
     private static String buildErrorMessage(String as, String value)
     {

data/src/main/java/org/embulk/filter/typecast/cast/TimestampCast.java

@@ -5,7 +5,10 @@ import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.msgpack.value.Value;
 
-public class TimestampCast {
+public class TimestampCast
+{
+    private TimestampCast() {}
+
     private static String buildErrorMessage(String as, Timestamp value)
     {
         return String.format("cannot cast Timestamp to %s: \"%s\"", as, value);
@@ -25,7 +28,7 @@ public class TimestampCast {
     {
         long epochSecond = value.getEpochSecond();
         long nano = value.getNano();
-        return epochSecond + ((double)nano / 1000000000.0);
+        return epochSecond + ((double) nano / 1000000000.0);
     }
 
     public static String asString(Timestamp value, TimestampFormatter formatter) throws DataException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-typecast
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-28 00:00:00.000000000 Z
+date: 2016-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,7 @@ files:
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
+- example/empty.yml
 - example/example.csv
 - example/example.yml
 - example/example2.yml
@@ -60,6 +61,7 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/typecast.rb
+- settings.gradle
 - src/main/java/org/embulk/filter/typecast/ColumnCaster.java
 - src/main/java/org/embulk/filter/typecast/ColumnVisitorImpl.java
 - src/main/java/org/embulk/filter/typecast/JsonCaster.java
@@ -72,7 +74,7 @@ files:
 - src/main/java/org/embulk/filter/typecast/cast/StringCast.java
 - src/main/java/org/embulk/filter/typecast/cast/TimestampCast.java
 - src/test/java/org/embulk/filter/TestTypecastFilterPlugin.java
-- classpath/embulk-filter-typecast-0.1.2.jar
+- classpath/embulk-filter-typecast-0.1.3.jar
 homepage: https://github.com/sonots/embulk-filter-typecast
 licenses:
 - MIT