embulk-filter-typecast 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db51b360ea765f3b96a301ec98144cc1f0033ebe
-  data.tar.gz: e9c35bc3f9af842a370b655bd81f8253d49d56e9
+  metadata.gz: e8a2482d0bd6fc6109bb8763f9a1f9db6b0733db
+  data.tar.gz: b80c4d01e823e3a4b59594002277fc78e1ee0ef9
 SHA512:
-  metadata.gz: 7dd4e6b33e658ad72e22b7ad65fe4738244505ac7ebd75d6b2333ff3f260bdd9ef979f5b800d3608836ff928fcd4e1d56e78290dab8cabc5c7a079660706097d
-  data.tar.gz: 6d356bbf862ad960684b65eda40e8bcae43f8559515cd1ee36468f92fb997f8a0e225c3b88dfe2ec0d00216e41bae65b494fed58046b3a36e90907965ed78f6a
+  metadata.gz: 7473b57b158d8936e14015358dcd21d3bfe852f0b7c33de10da4ad38aa4553b3ddf39e5b41c9ed11df32713c42a33506aba22820b2236590ab124035a1056783
+  data.tar.gz: 354bf8ab52c2ee5ba124af046abea9e311213c4ebaa03991f432504cc87094a0e1d4815c3c94434ee843de836f6e2146331540fd23b286071c000011bb64c55a

data/.gitignore

@@ -6,7 +6,6 @@
 /classpath/
 build/
 .idea
-*.csv
 .tags
 .ruby-version
 *.iml

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.2 (2016-04-28)
+
+Enhancements:
+
+* Support typecast in json path for casted json (from string)
+
 # 0.1.1 (2016-04-28)
 
 Enhancements:

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/example.csv

@@ -0,0 +1,11 @@
+timestamp,null,long,string,double,json1,json2,boolean
+2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"0"},{"long":0},true
+2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"1"},{"long":1},true
+2015-07-13 00:00:00.100000,,92,0hgDRI_m,810.9,{"string":"2"},{"long":2},true
+2015-07-13 00:00:00.100000,,93,KjCRAc-A,477.4,{"string":"3"},{"long":3},true
+2015-07-13 00:00:00.100000,,94,fyQVGlT8,725.3,{"string":"4"},{"long":4},true
+2015-07-13 00:00:00.100000,,95,FpBYRPWK,316.6,{"string":"5"},{"long":5},false
+2015-07-13 00:00:00.100000,,96,9ikvnUqp,369.5,{"string":"6"},{"long":6},false
+2015-07-13 00:00:00.100000,,97,RRNYDAzK,506.5,{"string":"7"},{"long":7},false
+2015-07-13 00:00:00.100000,,90,l6lTsvxd,903.4,{"string":"8"},{"long":8},false
+2015-07-13 00:00:00.100000,,91,XoALSEQg,394.5,{"string":"9"},{"long":9},false

data/example/example2.yml

@@ -14,7 +14,8 @@ in:
       - {name: long,      type: string}
       - {name: string,    type: string}
       - {name: double,    type: string}
-      - {name: json,      type: string}
+      - {name: json1,     type: string}
+      - {name: json2,     type: string}
       - {name: boolean,   type: boolean}
 filters:
   - type: typecast
@@ -24,7 +25,10 @@ filters:
       - {name: long,      type: long}
       - {name: string,    type: string}
       - {name: double,    type: double}
-      - {name: json,      type: json}
+      - {name: json1,     type: json}
+      - {name: json2,     type: json}
       - {name: boolean,   type: boolean}
+      - {name: "$.json1.string", type: long}
+      - {name: "$.json2.long",   type: long}
 out:
   type: "null"

data/src/main/java/org/embulk/filter/typecast/{TypecastPageBuilder.java → ColumnCaster.java}

@@ -2,17 +2,45 @@ package org.embulk.filter.typecast;
 
 import org.embulk.filter.typecast.cast.*;
 
+import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
+
 import org.embulk.spi.Column;
 import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
 import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
 import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.type.*;
 import org.msgpack.value.Value;
 
-class TypecastPageBuilder {
-    static void setFromBoolean(PageBuilder pageBuilder, Column outputColumn, boolean value) {
+import org.slf4j.Logger;
+
+
+class ColumnCaster
+{
+    private static final Logger logger = Exec.getLogger(TypecastFilterPlugin.class);
+    private final PluginTask task;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final PageReader pageReader;
+    private final PageBuilder pageBuilder;
+    private final JsonVisitor jsonVisitor;
+
+    ColumnCaster(TypecastFilterPlugin.PluginTask task, Schema inputSchema, Schema outputSchema,
+            PageReader pageReader, PageBuilder pageBuilder)
+    {
+        this.task = task;
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageReader = pageReader;
+        this.pageBuilder = pageBuilder;
+        this.jsonVisitor = new JsonVisitor(task, inputSchema, outputSchema);
+    }
+
+    public void setFromBoolean(Column outputColumn, boolean value) {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
             pageBuilder.setBoolean(outputColumn, BooleanCast.asBoolean(value));
@@ -31,7 +59,7 @@ class TypecastPageBuilder {
         }
     }
 
-    static void setFromLong(PageBuilder pageBuilder, Column outputColumn, long value)
+    public void setFromLong(Column outputColumn, long value)
     {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
@@ -51,7 +79,7 @@ class TypecastPageBuilder {
         }
     }
 
-    static void setFromDouble(PageBuilder pageBuilder, Column outputColumn, double value)
+    public void setFromDouble(Column outputColumn, double value)
     {
         try {
             Type outputType = outputColumn.getType();
@@ -76,7 +104,7 @@ class TypecastPageBuilder {
         }
     }
 
-    static void setFromString(PageBuilder pageBuilder, Column outputColumn, String value, TimestampParser timestampParser)
+    public void setFromString(Column outputColumn, String value, TimestampParser timestampParser)
     {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
@@ -90,13 +118,16 @@ class TypecastPageBuilder {
         } else if (outputType instanceof TimestampType) {
             pageBuilder.setTimestamp(outputColumn, StringCast.asTimestamp(value, timestampParser));
         } else if (outputType instanceof JsonType) {
-            pageBuilder.setJson(outputColumn, StringCast.asJson(value));
+            Value jsonValue = StringCast.asJson(value);
+            String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
+            Value castedValue = jsonVisitor.visit(jsonPath, jsonValue);
+            pageBuilder.setJson(outputColumn, castedValue);
         } else {
             assert(false);
         }
     }
 
-    static void setFromTimestamp(PageBuilder pageBuilder, Column outputColumn, Timestamp value, TimestampFormatter timestampFormatter)
+    public void setFromTimestamp(Column outputColumn, Timestamp value, TimestampFormatter timestampFormatter)
     {
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
@@ -116,21 +147,23 @@ class TypecastPageBuilder {
         }
     }
 
-    static void setFromJson(PageBuilder pageBuilder, Column outputColumn, Value value)
+    public void setFromJson(Column outputColumn, Value value)
     {
+        String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
+        Value castedValue = jsonVisitor.visit(jsonPath, value);
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {
-            pageBuilder.setBoolean(outputColumn, JsonCast.asBoolean(value));
+            pageBuilder.setBoolean(outputColumn, JsonCast.asBoolean(castedValue));
         } else if (outputType instanceof LongType) {
-            pageBuilder.setLong(outputColumn, JsonCast.asLong(value));
+            pageBuilder.setLong(outputColumn, JsonCast.asLong(castedValue));
         } else if (outputType instanceof DoubleType) {
-            pageBuilder.setDouble(outputColumn, JsonCast.asDouble(value));
+            pageBuilder.setDouble(outputColumn, JsonCast.asDouble(castedValue));
         } else if (outputType instanceof StringType) {
-            pageBuilder.setString(outputColumn, JsonCast.asString(value));
+            pageBuilder.setString(outputColumn, JsonCast.asString(castedValue));
         } else if (outputType instanceof TimestampType) {
-            pageBuilder.setTimestamp(outputColumn, JsonCast.asTimestamp(value));
+            pageBuilder.setTimestamp(outputColumn, JsonCast.asTimestamp(castedValue));
         } else if (outputType instanceof JsonType) {
-            pageBuilder.setJson(outputColumn, JsonCast.asJson(value));
+            pageBuilder.setJson(outputColumn, JsonCast.asJson(castedValue));
         } else {
             assert(false);
         }

data/src/main/java/org/embulk/filter/typecast/ColumnVisitorImpl.java

@@ -1,11 +1,8 @@
 package org.embulk.filter.typecast;
 
 import org.embulk.spi.*;
-import org.embulk.spi.type.Type;
-import org.msgpack.value.ArrayValue;
-import org.msgpack.value.MapValue;
-import org.msgpack.value.Value;
-import org.msgpack.value.ValueFactory;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.type.TimestampType;
 
 import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
 import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
@@ -16,8 +13,6 @@ import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 
 import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
 
 public class ColumnVisitorImpl
         implements ColumnVisitor
@@ -31,11 +26,10 @@ public class ColumnVisitorImpl
     private final HashMap<String, Column> outputColumnMap = new HashMap<>();
     private final HashMap<String, TimestampParser> timestampParserMap = new HashMap<>();
     private final HashMap<String, TimestampFormatter> timestampFormatterMap = new HashMap<>();
-    private final HashSet<String> shouldVisitJsonPathSet = new HashSet<>();
-    private final HashMap<String, Type> jsonPathTypeMap = new HashMap<>();
+    private final ColumnCaster columnCaster;
 
     ColumnVisitorImpl(PluginTask task, Schema inputSchema, Schema outputSchema,
-                      PageReader pageReader, PageBuilder pageBuilder)
+            PageReader pageReader, PageBuilder pageBuilder)
     {
         this.task         = task;
         this.inputSchema  = inputSchema;
@@ -43,11 +37,11 @@ public class ColumnVisitorImpl
         this.pageReader   = pageReader;
         this.pageBuilder  = pageBuilder;
 
+        this.columnCaster = new ColumnCaster(task, inputSchema, outputSchema, pageReader, pageBuilder);
+
         buildOutputColumnMap();
         buildTimestampParserMap();
         buildTimestampFormatterMap();
-        buildShouldVisitJsonPathSet();;
-        buildJsonPathTypeMap();
     }
 
     private void buildOutputColumnMap()
@@ -60,132 +54,46 @@ public class ColumnVisitorImpl
 
     private void buildTimestampParserMap()
     {
-        // columnName or jsonPath => TimestampParser
+        // columnName => TimestampParser
         for (ColumnConfig columnConfig : task.getColumns()) {
-            TimestampParser parser = getTimestampParser(columnConfig, task);
-            this.timestampParserMap.put(columnConfig.getName(), parser);
-        }
-    }
-
-    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
-    {
-        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
-        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
-        return new TimestampParser(task.getJRuby(), format, timezone);
-    }
-
-    private void buildTimestampFormatterMap()
-    {
-        // columnName or jsonPath => TimestampFormatter
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
-            this.timestampFormatterMap.put(columnConfig.getName(), parser);
-        }
-    }
-
-    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
-    {
-        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
-        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
-        return new TimestampFormatter(task.getJRuby(), format, timezone);
-    }
-
-    private void buildShouldVisitJsonPathSet()
-    {
-        // json partial path => Boolean to avoid unnecessary type: json visit
-        for (ColumnConfig columnConfig : task.getColumns()) {
-            String name = columnConfig.getName();
-            if (!name.startsWith("$.")) {
-                continue;
+            if (columnConfig.getName().startsWith("$.")) {
+                continue; // type: json columns do not support type: timestamp
             }
-            String[] parts = name.split("\\.");
-            StringBuilder partialPath = new StringBuilder("$");
-            for (int i = 1; i < parts.length; i++) {
-                if (parts[i].contains("[")) {
-                    String[] arrayParts = parts[i].split("\\[");
-                    partialPath.append(".").append(arrayParts[0]);
-                    this.shouldVisitJsonPathSet.add(partialPath.toString());
-                    for (int j = 1; j < arrayParts.length; j++) {
-                        partialPath.append("[").append(arrayParts[j]);
-                        this.shouldVisitJsonPathSet.add(partialPath.toString());
-                    }
-                }
-                else {
-                    partialPath.append(".").append(parts[i]);
-                    this.shouldVisitJsonPathSet.add(partialPath.toString());
-                }
+            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
+            if (inputColumn.getType() instanceof StringType && columnConfig.getType() instanceof TimestampType) {
+                TimestampParser parser = getTimestampParser(columnConfig, task);
+                this.timestampParserMap.put(columnConfig.getName(), parser);
             }
         }
     }
 
-    private void buildJsonPathTypeMap()
+    private void buildTimestampFormatterMap()
     {
-        // json path => Type
+        // columnName => TimestampFormatter
         for (ColumnConfig columnConfig : task.getColumns()) {
-            String name = columnConfig.getName();
-            if (!name.startsWith("$.")) {
-                continue;
+            if (columnConfig.getName().startsWith("$.")) {
+                continue; // type: json columns do not have type: timestamp
+            }
+            Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
+            if (inputColumn.getType() instanceof TimestampType && columnConfig.getType() instanceof StringType) {
+                TimestampFormatter parser = getTimestampFormatter(columnConfig, task);
+                this.timestampFormatterMap.put(columnConfig.getName(), parser);
             }
-            Type type = columnConfig.getType();
-            this.jsonPathTypeMap.put(name, type);
         }
     }
 
-    private boolean shouldVisitJsonPath(String jsonPath)
+    private TimestampParser getTimestampParser(ColumnConfig columnConfig, PluginTask task)
     {
-        return shouldVisitJsonPathSet.contains(jsonPath);
+        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
+        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
+        return new TimestampParser(task.getJRuby(), format, timezone);
     }
 
-    private Value castJsonRecursively(PluginTask task, String jsonPath, Value value)
+    private TimestampFormatter getTimestampFormatter(ColumnConfig columnConfig, PluginTask task)
     {
-        if (!shouldVisitJsonPath(jsonPath)) {
-            return value;
-        }
-        if (value.isArrayValue()) {
-            ArrayValue arrayValue = value.asArrayValue();
-            int size = arrayValue.size();
-            Value[] newValue = new Value[size];
-            for (int i = 0; i < size; i++) {
-                String k = new StringBuilder(jsonPath).append("[").append(Integer.toString(i)).append("]").toString();
-                Value v = arrayValue.get(i);
-                newValue[i] = castJsonRecursively(task, k, v);
-            }
-            return ValueFactory.newArray(newValue, true);
-        }
-        else if (value.isMapValue()) {
-            MapValue mapValue = value.asMapValue();
-            int size = mapValue.size() * 2;
-            Value[] newValue = new Value[size];
-            int i = 0;
-            for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
-                Value k = entry.getKey();
-                Value v = entry.getValue();
-                String newPath = new StringBuilder(jsonPath).append(".").append(k.asStringValue().asString()).toString();
-                Value r = castJsonRecursively(task, newPath, v);
-                newValue[i++] = k;
-                newValue[i++] = r;
-            }
-            return ValueFactory.newMap(newValue, true);
-        }
-        else if (value.isBooleanValue()) {
-            Type outputType = jsonPathTypeMap.get(jsonPath);
-            return TypecastJsonBuilder.getFromBoolean(outputType, value.asBooleanValue().getBoolean());
-        }
-        else if (value.isIntegerValue()) {
-            Type outputType = jsonPathTypeMap.get(jsonPath);
-            return TypecastJsonBuilder.getFromLong(outputType, value.asIntegerValue().asLong());
-        }
-        else if (value.isFloatValue()) {
-            Type outputType = jsonPathTypeMap.get(jsonPath);
-            return TypecastJsonBuilder.getFromDouble(outputType, value.asFloatValue().toDouble());
-        }
-        else if (value.isStringValue()) {
-            Type outputType = jsonPathTypeMap.get(jsonPath);
-            return TypecastJsonBuilder.getFromString(outputType, value.asStringValue().asString());
-        }
-        else {
-            return value;
-        }
+        String format = columnConfig.getFormat().or(task.getDefaultTimestampFormat());
+        DateTimeZone timezone = columnConfig.getTimeZone().or(task.getDefaultTimeZone());
+        return new TimestampFormatter(task.getJRuby(), format, timezone);
     }
 
     private interface PageBuildable
@@ -217,7 +125,7 @@ public class ColumnVisitorImpl
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromBoolean(pageBuilder, outputColumn, pageReader.getBoolean(inputColumn));
+                columnCaster.setFromBoolean(outputColumn, pageReader.getBoolean(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -229,7 +137,7 @@ public class ColumnVisitorImpl
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromLong(pageBuilder, outputColumn, pageReader.getLong(inputColumn));
+                columnCaster.setFromLong(outputColumn, pageReader.getLong(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -241,7 +149,7 @@ public class ColumnVisitorImpl
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromDouble(pageBuilder, outputColumn, pageReader.getDouble(inputColumn));
+                columnCaster.setFromDouble(outputColumn, pageReader.getDouble(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -254,8 +162,7 @@ public class ColumnVisitorImpl
         final TimestampParser timestampParser = timestampParserMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromString(
-                        pageBuilder, outputColumn, pageReader.getString(inputColumn), timestampParser);
+                columnCaster.setFromString(outputColumn, pageReader.getString(inputColumn), timestampParser);
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -268,8 +175,7 @@ public class ColumnVisitorImpl
         final TimestampFormatter timestampFormatter = timestampFormatterMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromTimestamp(
-                        pageBuilder, outputColumn, pageReader.getTimestamp(inputColumn), timestampFormatter);
+                columnCaster.setFromTimestamp(outputColumn, pageReader.getTimestamp(inputColumn), timestampFormatter);
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);
@@ -278,13 +184,10 @@ public class ColumnVisitorImpl
     @Override
     public void jsonColumn(final Column inputColumn)
     {
-        String jsonPath = new StringBuilder("$.").append(inputColumn.getName()).toString();
-        Value value = pageReader.getJson(inputColumn);
-        final Value castedValue = castJsonRecursively(task, jsonPath, value);
         final Column outputColumn = outputColumnMap.get(inputColumn.getName());
         PageBuildable op = new PageBuildable() {
             public void run() throws DataException {
-                TypecastPageBuilder.setFromJson(pageBuilder, outputColumn, castedValue);
+                columnCaster.setFromJson(outputColumn, pageReader.getJson(inputColumn));
             }
         };
         withStopOnInvalidRecord(op, inputColumn, outputColumn);

data/src/main/java/org/embulk/filter/typecast/{TypecastJsonBuilder.java → JsonCaster.java}

@@ -4,21 +4,28 @@ import org.embulk.filter.typecast.cast.*;
 
 import org.embulk.spi.DataException;
 import org.embulk.spi.type.*;
+import org.msgpack.value.BooleanValue;
+import org.msgpack.value.IntegerValue;
+import org.msgpack.value.FloatValue;
+import org.msgpack.value.StringValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
 
-class TypecastJsonBuilder {
-    static Value getFromBoolean(Type outputType, boolean value) {
+class JsonCaster
+{
+    public JsonCaster()
+    {
+    }
+
+    public Value fromBoolean(Type outputType, BooleanValue value) {
         if (outputType instanceof BooleanType) {
-            return ValueFactory.newBoolean(value);
+            return value;
         } else if (outputType instanceof LongType) {
-            return ValueFactory.newInteger(BooleanCast.asLong(value));
+            return ValueFactory.newInteger(BooleanCast.asLong(value.getBoolean()));
         } else if (outputType instanceof DoubleType) {
-            return ValueFactory.newFloat(BooleanCast.asDouble(value));
+            return ValueFactory.newFloat(BooleanCast.asDouble(value.getBoolean()));
         } else if (outputType instanceof StringType) {
-            return ValueFactory.newString(BooleanCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
-            throw new DataException(String.format("no timestamp type in json: \"%s\"", value));
+            return ValueFactory.newString(BooleanCast.asString(value.getBoolean()));
         } else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast boolean to json: \"%s\"", value));
         } else {
@@ -27,18 +34,16 @@ class TypecastJsonBuilder {
         }
     }
 
-    static Value getFromLong(Type outputType, long value)
+    public Value fromLong(Type outputType, IntegerValue value)
     {
         if (outputType instanceof BooleanType) {
-            return ValueFactory.newBoolean(LongCast.asBoolean(value));
+            return ValueFactory.newBoolean(LongCast.asBoolean(value.asLong()));
         } else if (outputType instanceof LongType) {
-            return ValueFactory.newInteger(value);
+            return value;
         } else if (outputType instanceof DoubleType) {
-            return ValueFactory.newFloat(LongCast.asDouble(value));
+            return ValueFactory.newFloat(LongCast.asDouble(value.asLong()));
         } else if (outputType instanceof StringType) {
-            return ValueFactory.newString(LongCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
-            throw new DataException(String.format("no timestamp type in json: \"%s\"", value));
+            return ValueFactory.newString(LongCast.asString(value.asLong()));
         } else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast long to json:: \"%s\"", value));
         } else {
@@ -47,18 +52,16 @@ class TypecastJsonBuilder {
         }
     }
 
-    static Value getFromDouble(Type outputType, double value)
+    public Value fromDouble(Type outputType, FloatValue value)
     {
         if (outputType instanceof BooleanType) {
-            return ValueFactory.newBoolean(DoubleCast.asBoolean(value));
+            return ValueFactory.newBoolean(DoubleCast.asBoolean(value.toDouble()));
         } else if (outputType instanceof LongType) {
-            return ValueFactory.newInteger(DoubleCast.asLong(value));
+            return ValueFactory.newInteger(DoubleCast.asLong(value.toDouble()));
         } else if (outputType instanceof DoubleType) {
-            return ValueFactory.newFloat(DoubleCast.asDouble(value));
+            return value;
         } else if (outputType instanceof StringType) {
-            return ValueFactory.newString(DoubleCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
-            throw new DataException(String.format("no timestamp type in json: \"%s\"", value));
+            return ValueFactory.newString(DoubleCast.asString(value.toDouble()));
         } else if (outputType instanceof JsonType) {
             throw new DataException(String.format("cannot cast double to json:: \"%s\"", value));
         } else {
@@ -67,40 +70,18 @@ class TypecastJsonBuilder {
         }
     }
 
-    static Value getFromString(Type outputType, String value)
-    {
-        if (outputType instanceof BooleanType) {
-            return ValueFactory.newBoolean(StringCast.asBoolean(value));
-        } else if (outputType instanceof LongType) {
-            return ValueFactory.newInteger(StringCast.asLong(value));
-        } else if (outputType instanceof DoubleType) {
-            return ValueFactory.newFloat(StringCast.asDouble(value));
-        } else if (outputType instanceof StringType) {
-            return ValueFactory.newString(StringCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
-            throw new DataException(String.format("no timestamp type in json: \"%s\"", value));
-        } else if (outputType instanceof JsonType) {
-            return StringCast.asJson(value);
-        } else {
-            assert(false);
-            return null;
-        }
-    }
-
-    static Value getFromJson(Type outputType, Value value)
+    public Value fromString(Type outputType, StringValue value)
     {
         if (outputType instanceof BooleanType) {
-            return ValueFactory.newBoolean(JsonCast.asBoolean(value));
+            return ValueFactory.newBoolean(StringCast.asBoolean(value.asString()));
         } else if (outputType instanceof LongType) {
-            return ValueFactory.newInteger(JsonCast.asLong(value));
+            return ValueFactory.newInteger(StringCast.asLong(value.asString()));
         } else if (outputType instanceof DoubleType) {
-            return ValueFactory.newFloat(JsonCast.asDouble(value));
+            return ValueFactory.newFloat(StringCast.asDouble(value.asString()));
         } else if (outputType instanceof StringType) {
-            return ValueFactory.newString(JsonCast.asString(value));
-        } else if (outputType instanceof TimestampType) {
-            throw new DataException(String.format("no timestamp type in json: \"%s\"", value));
-        } else if (outputType instanceof JsonType) {
             return value;
+        } else if (outputType instanceof JsonType) {
+            return StringCast.asJson(value.asString());
         } else {
             assert(false);
             return null;

data/src/main/java/org/embulk/filter/typecast/JsonVisitor.java

@@ -0,0 +1,136 @@
+package org.embulk.filter.typecast;
+
+import org.embulk.spi.*;
+import org.embulk.spi.type.Type;
+import org.msgpack.value.ArrayValue;
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
+import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
+
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+
+public class JsonVisitor
+{
+    private static final Logger logger = Exec.getLogger(TypecastFilterPlugin.class);
+    private final PluginTask task;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final HashSet<String> shouldVisitSet = new HashSet<>();
+    private final HashMap<String, Type> jsonPathTypeMap = new HashMap<>();
+    private final JsonCaster jsonCaster = new JsonCaster();
+
+    JsonVisitor(PluginTask task, Schema inputSchema, Schema outputSchema)
+    {
+        this.task         = task;
+        this.inputSchema  = inputSchema;
+        this.outputSchema = outputSchema;
+
+        buildShouldVisitSet();
+        buildJsonPathTypeMap();
+    }
+
+    private void buildJsonPathTypeMap()
+    {
+        // json path => Type
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            String name = columnConfig.getName();
+            if (!name.startsWith("$.")) {
+                continue;
+            }
+            Type type = columnConfig.getType();
+            this.jsonPathTypeMap.put(name, type);
+        }
+    }
+
+    private void buildShouldVisitSet()
+    {
+        // json partial path => Boolean to avoid unnecessary type: json visit
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            String name = columnConfig.getName();
+            if (!name.startsWith("$.")) {
+                continue;
+            }
+            String[] parts = name.split("\\.");
+            StringBuilder partialPath = new StringBuilder("$");
+            for (int i = 1; i < parts.length; i++) {
+                if (parts[i].contains("[")) {
+                    String[] arrayParts = parts[i].split("\\[");
+                    partialPath.append(".").append(arrayParts[0]);
+                    this.shouldVisitSet.add(partialPath.toString());
+                    for (int j = 1; j < arrayParts.length; j++) {
+                        partialPath.append("[").append(arrayParts[j]);
+                        this.shouldVisitSet.add(partialPath.toString());
+                    }
+                }
+                else {
+                    partialPath.append(".").append(parts[i]);
+                    this.shouldVisitSet.add(partialPath.toString());
+                }
+            }
+        }
+    }
+
+    private boolean shouldVisit(String jsonPath)
+    {
+        return shouldVisitSet.contains(jsonPath);
+    }
+
+    public Value visit(String jsonPath, Value value)
+    {
+        if (!shouldVisit(jsonPath)) {
+            return value;
+        }
+        if (value.isArrayValue()) {
+            ArrayValue arrayValue = value.asArrayValue();
+            int size = arrayValue.size();
+            Value[] newValue = new Value[size];
+            for (int i = 0; i < size; i++) {
+                String k = new StringBuilder(jsonPath).append("[").append(Integer.toString(i)).append("]").toString();
+                Value v = arrayValue.get(i);
+                newValue[i] = visit(k, v);
+            }
+            return ValueFactory.newArray(newValue, true);
+        }
+        else if (value.isMapValue()) {
+            MapValue mapValue = value.asMapValue();
+            int size = mapValue.size() * 2;
+            Value[] newValue = new Value[size];
+            int i = 0;
+            for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
+                Value k = entry.getKey();
+                Value v = entry.getValue();
+                String newPath = new StringBuilder(jsonPath).append(".").append(k.asStringValue().asString()).toString();
+                Value r = visit(newPath, v);
+                newValue[i++] = k;
+                newValue[i++] = r;
+            }
+            return ValueFactory.newMap(newValue, true);
+        }
+        else if (value.isBooleanValue()) {
+            Type outputType = jsonPathTypeMap.get(jsonPath);
+            return jsonCaster.fromBoolean(outputType, value.asBooleanValue());
+        }
+        else if (value.isIntegerValue()) {
+            Type outputType = jsonPathTypeMap.get(jsonPath);
+            return jsonCaster.fromLong(outputType, value.asIntegerValue());
+        }
+        else if (value.isFloatValue()) {
+            Type outputType = jsonPathTypeMap.get(jsonPath);
+            return jsonCaster.fromDouble(outputType, value.asFloatValue());
+        }
+        else if (value.isStringValue()) {
+            Type outputType = jsonPathTypeMap.get(jsonPath);
+            return jsonCaster.fromString(outputType, value.asStringValue());
+        }
+        else {
+            return value;
+        }
+    }
+}

data/src/main/java/org/embulk/filter/typecast/TypecastFilterPlugin.java

@@ -4,15 +4,18 @@ import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 
 import org.embulk.spi.*;
+import org.embulk.spi.time.Timestamp;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.time.TimestampParser;
 
+import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
 import org.joda.time.DateTimeZone;
 import org.jruby.embed.ScriptingContainer;
@@ -86,7 +89,7 @@ public class TypecastFilterPlugin implements FilterPlugin
         // throw if column does not exist
         for (ColumnConfig columnConfig : columnConfigs) {
             String name = columnConfig.getName();
-            if (name.startsWith("$.")) {
+            if (name.startsWith("$.")) { // check only top level column name
                 String firstName = name.split("\\.", 3)[1];
                 inputSchema.lookupColumn(firstName);
             }
@@ -94,6 +97,13 @@ public class TypecastFilterPlugin implements FilterPlugin
                 inputSchema.lookupColumn(name);
             }
         }
+        // throw if timestamp is specified in json path
+        for (ColumnConfig columnConfig : columnConfigs) {
+            String name = columnConfig.getName();
+            if (name.startsWith("$.") && columnConfig.getType() instanceof TimestampType) {
+                throw new ConfigException(String.format("embulk-filter-typecast: timestamp type is not supported in json column: \"%s\"", name));
+            }
+        }
     }
 
     private Schema buildOuputSchema(final PluginTask task, final Schema inputSchema)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-typecast
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-27 00:00:00.000000000 Z
+date: 2016-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,6 +52,7 @@ files:
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
+- example/example.csv
 - example/example.yml
 - example/example2.yml
 - gradle/wrapper/gradle-wrapper.jar
@@ -59,10 +60,11 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/filter/typecast.rb
+- src/main/java/org/embulk/filter/typecast/ColumnCaster.java
 - src/main/java/org/embulk/filter/typecast/ColumnVisitorImpl.java
+- src/main/java/org/embulk/filter/typecast/JsonCaster.java
+- src/main/java/org/embulk/filter/typecast/JsonVisitor.java
 - src/main/java/org/embulk/filter/typecast/TypecastFilterPlugin.java
-- src/main/java/org/embulk/filter/typecast/TypecastJsonBuilder.java
-- src/main/java/org/embulk/filter/typecast/TypecastPageBuilder.java
 - src/main/java/org/embulk/filter/typecast/cast/BooleanCast.java
 - src/main/java/org/embulk/filter/typecast/cast/DoubleCast.java
 - src/main/java/org/embulk/filter/typecast/cast/JsonCast.java
@@ -70,7 +72,7 @@ files:
 - src/main/java/org/embulk/filter/typecast/cast/StringCast.java
 - src/main/java/org/embulk/filter/typecast/cast/TimestampCast.java
 - src/test/java/org/embulk/filter/TestTypecastFilterPlugin.java
-- classpath/embulk-filter-typecast-0.1.1.jar
+- classpath/embulk-filter-typecast-0.1.2.jar
 homepage: https://github.com/sonots/embulk-filter-typecast
 licenses:
 - MIT