embulk-filter-column 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 485925060323de88d131e5d28c6646faa7c1d066
-  data.tar.gz: 44ac8c06b82a3d893bb3e8a9f0f1f33ebd847984
+  metadata.gz: 1faf79742c3c3eefadac1c9c31884e335df93761
+  data.tar.gz: 1b3b85f3b54c51bcc26b8dde92f6c7e8c5728238
 SHA512:
-  metadata.gz: 5952f424c3b9f40cdb0929c63860d6c27faf16c611313ad13f159adf30372e94a7fd0e93b7674f401fb6345829cd357a6d1096f0fb56f391c0d07007385812ec
-  data.tar.gz: 8676e815498a52bb0a25b0ccb2918ee08de2a2c8506b1007da03e40bf5a1db5d85a826c21412d645d3455900e5259dbb148b98711d1c4f102457fc41c7527139
+  metadata.gz: 6a5cbcd1e4b1dbf79fcdd66dc5220f6e9023c9416880824fa4f3de0d13b3685e8243a3104bb427f0c222eb7c1ac17f021786aaf33b9564ee5c487d9218faafa3
+  data.tar.gz: 181ea5b27402a4d95482fade22bba2a136eb69d54808289a44b4790bcfb1d345e402b063663df36f45180082a39fa318e8cda82eef2192f7b16b9bc528c6a428

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.1 (2016-06-03)
+
+Fixes:
+
+* Fix JSONPath support
+
 # 0.5.0 (2016-05-31)
 
 Enhancements:

data/README.md

@@ -151,6 +151,14 @@ Run test:
 $ ./gradlew test
 ```
 
+Run test with coverage reports:
+
+```
+$ ./gradlew test jacocoTestReport
+```
+
+open build/reports/jacoco/test/html/index.html
+
 Run checkstyle:
 
 ```

data/build.gradle

@@ -3,6 +3,7 @@ plugins {
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -13,15 +14,17 @@ configurations {
     provided
 }
 
-version = "0.5.0"
+version = "0.5.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.2"
-    provided "org.embulk:embulk-core:0.8.2"
-    // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
+    compile  "org.embulk:embulk-core:0.8.+"
+    provided "org.embulk:embulk-core:0.8.+"
+
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.+:tests"
+    testCompile "org.embulk:embulk-standards:0.8.+"
 }
 
 checkstyle {

data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java

@@ -106,7 +106,7 @@ public class ColumnFilterPlugin implements FilterPlugin
         }
     }
 
-    private Schema buildOutputSchema(PluginTask task, Schema inputSchema)
+    static Schema buildOutputSchema(PluginTask task, Schema inputSchema)
     {
         List<ColumnConfig> columns = task.getColumns();
         List<ColumnConfig> addColumns = task.getAddColumns();

data/src/main/java/org/embulk/filter/column/JsonColumn.java

@@ -2,6 +2,7 @@ package org.embulk.filter.column;
 
 import org.embulk.config.ConfigException;
 import org.embulk.spi.type.Type;
+import org.msgpack.value.IntegerValue;
 import org.msgpack.value.StringValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
@@ -16,12 +17,14 @@ public class JsonColumn
     private StringValue pathValue = null;
     private String parentPath = null;
     private String baseName = null;
+    private Long baseIndex = null;
     private StringValue parentPathValue = null;
     private StringValue baseNameValue = null;
 
     private StringValue srcValue = null;
     private String srcParentPath = null;
     private String srcBaseName = null;
+    private Long srcBaseIndex = null;
     private StringValue srcParentPathValue = null;
     private StringValue srcBaseNameValue = null;
 
@@ -45,12 +48,14 @@ public class JsonColumn
         this.pathValue = ValueFactory.newString(path);
         this.parentPath = parentPath(path);
         this.baseName = baseName(path);
+        this.baseIndex = baseIndex(path);
         this.parentPathValue = ValueFactory.newString(parentPath);
         this.baseNameValue = ValueFactory.newString(baseName);
 
         this.srcValue = ValueFactory.newString(this.src);
         this.srcParentPath = parentPath(this.src);
         this.srcBaseName = baseName(this.src);
+        this.srcBaseIndex = baseIndex(this.src);
         this.srcParentPathValue = ValueFactory.newString(this.srcParentPath);
         this.srcBaseNameValue = ValueFactory.newString(this.srcBaseName);
 
@@ -94,6 +99,11 @@ public class JsonColumn
         return baseName;
     }
 
+    public Long getBaseIndex()
+    {
+        return baseIndex;
+    }
+
     public StringValue getParentPathValue()
     {
         return parentPathValue;
@@ -119,6 +129,11 @@ public class JsonColumn
         return srcBaseName;
     }
 
+    public Long getSrcBaseIndex()
+    {
+        return srcBaseIndex;
+    }
+
     public StringValue getSrcParentPathValue()
     {
         return srcParentPathValue;
@@ -134,8 +149,7 @@ public class JsonColumn
     {
         String[] parts = path.split("\\.");
         StringBuilder builder = new StringBuilder();
-        builder.append(parts[0]);
-        for (int i = 1; i < parts.length - 1; i++) {
+        for (int i = 0; i < parts.length - 1; i++) {
             builder.append(".").append(parts[i]);
         }
         if (parts[parts.length - 1].contains("[")) {
@@ -145,12 +159,35 @@ public class JsonColumn
                 builder.append("[").append(arrayParts[j]);
             }
         }
-        return builder.toString();
+        return builder.deleteCharAt(0).toString();
     }
 
     public static String baseName(String path)
     {
         String[] parts = path.split("\\.");
-        return parts[parts.length - 1];
+        String[] arrayParts = parts[parts.length - 1].split("\\[");
+        if (arrayParts.length == 1) { // no [i]
+            return arrayParts[arrayParts.length - 1];
+        }
+        else {
+            return "[" + arrayParts[arrayParts.length - 1];
+        }
+    }
+
+    public static Long baseIndex(String path)
+    {
+        String baseName = baseName(path);
+        if (baseName.startsWith("[") && baseName.endsWith("]")) {
+            String baseIndex = baseName.substring(1, baseName.length() - 1);
+            try {
+                return Long.parseLong(baseIndex);
+            }
+            catch (NumberFormatException e) {
+                return null;
+            }
+        }
+        else {
+            return null;
+        }
     }
 }

data/src/main/java/org/embulk/filter/column/JsonVisitor.java

@@ -30,14 +30,14 @@ import java.util.Map;
 
 public class JsonVisitor
 {
-    private static final Logger logger = Exec.getLogger(ColumnFilterPlugin.class);
-    private final PluginTask task;
-    private final Schema inputSchema;
-    private final Schema outputSchema;
-    private final HashSet<String> shouldVisitSet = new HashSet<>();
-    private final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonColumns = new HashMap<>();
-    private final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonAddColumns = new HashMap<>();
-    private final HashMap<String, HashSet<String>> jsonDropColumns = new HashMap<>();
+    static final Logger logger = Exec.getLogger(ColumnFilterPlugin.class);
+    final PluginTask task;
+    final Schema inputSchema;
+    final Schema outputSchema;
+    final HashSet<String> shouldVisitSet = new HashSet<>();
+    final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonColumns = new HashMap<>();
+    final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonAddColumns = new HashMap<>();
+    final HashMap<String, HashSet<String>> jsonDropColumns = new HashMap<>();
 
     JsonVisitor(PluginTask task, Schema inputSchema, Schema outputSchema)
     {
@@ -132,7 +132,7 @@ public class JsonVisitor
                 }
                 if (column.getSrc().isPresent()) {
                     String src = column.getSrc().get();
-                    jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
+                    jsonColumnsPut(name, new JsonColumn(name, null, null, src));
                 }
                 else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
                     Type type = column.getType().get();
@@ -190,7 +190,7 @@ public class JsonVisitor
                     partialPath.append(".").append(arrayParts[0]);
                     this.shouldVisitSet.add(partialPath.toString());
                     for (int j = 1; j < arrayParts.length; j++) {
-                        // Supports both [0] and [*]
+                        // Simply add [0] or [*] here
                         partialPath.append("[").append(arrayParts[j]);
                         this.shouldVisitSet.add(partialPath.toString());
                     }
@@ -234,28 +234,43 @@ public class JsonVisitor
             for (int i = 0; i < size; i++) {
                 String newPath = newArrayJsonPath(rootPath, i);
                 if (! jsonDropColumns.contains(newPath)) {
-                    newValue.add(j++, visit(newPath, arrayValue.get(i)));
+                    Value v = arrayValue.get(i);
+                    newValue.add(j++, visit(newPath, v));
                 }
             }
         }
         else if (this.jsonColumns.containsKey(rootPath)) {
-            LinkedHashMap<String, JsonColumn> jsonColumns = this.jsonColumns.get(rootPath);
-            for (int i = 0; i < size; i++) {
-                String newPath = newArrayJsonPath(rootPath, i);
-                if (jsonColumns.containsKey(newPath)) {
-                    newValue.add(j++, visit(newPath, arrayValue.get(i)));
+            for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
+                int src = jsonColumn.getSrcBaseIndex().intValue();
+                Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
+                if (v == null) {
+                    v = jsonColumn.getDefaultValue();
                 }
+                String newPath = jsonColumn.getPath();
+                Value visited = visit(newPath, v);
+                // int i = jsonColumn.getBaseIndex().intValue();
+                // index is shifted, so j++ is used.
+                newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
             }
         }
         else {
             for (int i = 0; i < size; i++) {
                 String newPath = newArrayJsonPath(rootPath, i);
-                newValue.add(j++, visit(newPath, arrayValue.get(i)));
+                Value v = arrayValue.get(i);
+                newValue.add(j++, visit(newPath, v));
             }
         }
         if (this.jsonAddColumns.containsKey(rootPath)) {
             for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
-                newValue.add(j++, jsonColumn.getDefaultValue());
+                int src = jsonColumn.getSrcBaseIndex().intValue();
+                Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
+                if (v == null) {
+                    v = jsonColumn.getDefaultValue();
+                }
+                String newPath = jsonColumn.getPath();
+                Value visited = visit(newPath, v);
+                // this ignores specified index, but appends to last now
+                newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
             }
         }
         return ValueFactory.newArray(newValue.toArray(new Value[0]), true);
@@ -281,17 +296,16 @@ public class JsonVisitor
         }
         else if (this.jsonColumns.containsKey(rootPath)) {
             Map<Value, Value> map = mapValue.map();
-            LinkedHashMap<String, JsonColumn> jsonColumns = this.jsonColumns.get(rootPath);
-            for (JsonColumn jsonColumn : jsonColumns.values()) {
+            for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
                 Value src = jsonColumn.getSrcBaseNameValue();
                 Value v = map.get(src);
+                if (v == null) {
+                    v = jsonColumn.getDefaultValue();
+                }
                 String newPath = jsonColumn.getPath();
                 Value visited = visit(newPath, v);
-                if (visited == null) {
-                    visited = jsonColumn.getDefaultValue();
-                }
-                newValue.add(i++, jsonColumn.getPathValue());
-                newValue.add(i++, visited);
+                newValue.add(i++, jsonColumn.getBaseNameValue());
+                newValue.add(i++, visited == null ? ValueFactory.newNil() : visited);
             }
         }
         else {
@@ -306,15 +320,16 @@ public class JsonVisitor
         }
         if (this.jsonAddColumns.containsKey(rootPath)) {
             Map<Value, Value> map = mapValue.map();
-            LinkedHashMap<String, JsonColumn> jsonAddColumns = this.jsonAddColumns.get(rootPath);
-            for (JsonColumn jsonColumn : jsonAddColumns.values()) {
+            for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
                 Value src = jsonColumn.getSrcBaseNameValue();
                 Value v = map.get(src);
                 if (v == null) {
                     v = jsonColumn.getDefaultValue();
                 }
-                newValue.add(i++, jsonColumn.getPathValue());
-                newValue.add(i++, v);
+                String newPath = jsonColumn.getPath();
+                Value visited = visit(newPath, v);
+                newValue.add(i++, jsonColumn.getBaseNameValue());
+                newValue.add(i++, visited == null ? ValueFactory.newNil() : visited);
             }
         }
         return ValueFactory.newMap(newValue.toArray(new Value[0]), true);
@@ -325,7 +340,10 @@ public class JsonVisitor
         if (! shouldVisit(rootPath)) {
             return value;
         }
-        if (value.isArrayValue()) {
+        if (value == null) {
+            return null;
+        }
+        else if (value.isArrayValue()) {
             return visitArray(rootPath, value.asArrayValue());
         }
         else if (value.isMapValue()) {

data/src/test/java/org/embulk/filter/column/TestJsonColumn.java

@@ -0,0 +1,57 @@
+package org.embulk.filter.column;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.embulk.spi.type.Types;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+public class TestJsonColumn {
+    @Test
+    public void initialize()
+    {
+        try {
+            JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN);
+            assertEquals("$.foo.bar", column.getSrc());
+            assertEquals(ValueFactory.newNil(), column.getDefaultValue());
+        }
+        catch (Exception e) {
+            fail();
+        }
+
+        try {
+            Value defaultValue = ValueFactory.newBoolean(true);
+            JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN, defaultValue);
+            assertEquals("$.foo.bar", column.getSrc());
+            assertEquals(defaultValue, column.getDefaultValue());
+        }
+        catch (Exception e) {
+            fail();
+        }
+    }
+
+    @Test
+    public void parentPath()
+    {
+        assertEquals("$.foo.bar", JsonColumn.parentPath("$.foo.bar.baz"));
+        assertEquals("$.foo", JsonColumn.parentPath("$.foo.bar"));
+        assertEquals("$", JsonColumn.parentPath("$.foo"));
+        assertEquals("$.foo[0]", JsonColumn.parentPath("$.foo[0][1]"));
+        assertEquals("$.foo", JsonColumn.parentPath("$.foo[0]"));
+        assertEquals("$", JsonColumn.parentPath("$[0]"));
+    }
+
+    @Test
+    public void baseName()
+    {
+        assertEquals("baz", JsonColumn.baseName("$.foo.bar.baz"));
+        assertEquals("bar", JsonColumn.baseName("$.foo.bar"));
+        assertEquals("foo", JsonColumn.baseName("$.foo"));
+        assertEquals("[1]", JsonColumn.baseName("$.foo[0][1]"));
+        assertEquals("[0]", JsonColumn.baseName("$.foo[0]"));
+        assertEquals("[0]", JsonColumn.baseName("$[0]"));
+    }
+}

data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java

@@ -0,0 +1,408 @@
+package org.embulk.filter.column;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.Lists;
+import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
+import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.type.Type;
+import org.joda.time.DateTimeZone;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static junit.framework.TestCase.assertFalse;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.msgpack.value.MapValue;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueFactory;
+
+import static org.embulk.spi.type.Types.BOOLEAN;
+import static org.embulk.spi.type.Types.DOUBLE;
+import static org.embulk.spi.type.Types.JSON;
+import static org.embulk.spi.type.Types.LONG;
+import static org.embulk.spi.type.Types.STRING;
+import static org.embulk.spi.type.Types.TIMESTAMP;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+public class TestJsonVisitor {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Before
+    public void createReasource()
+    {
+        // config = config().set("type", "column");
+    }
+
+    private ConfigSource config()
+    {
+        return runtime.getExec().newConfigSource();
+    }
+
+    private Schema schema(Column... columns)
+    {
+        return new Schema(Lists.newArrayList(columns));
+    }
+
+    private ConfigSource configFromYamlString(String... lines)
+    {
+        StringBuilder builder = new StringBuilder();
+        for (String line : lines) {
+            builder.append(line).append("\n");
+        }
+        String yamlString = builder.toString();
+
+        ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
+        return loader.fromYamlString(yamlString);
+    }
+
+    private PluginTask taskFromYamlString(String... lines)
+    {
+        ConfigSource config = configFromYamlString(lines);
+        return config.loadConfig(PluginTask.class);
+    }
+
+    private JsonVisitor jsonVisitor(PluginTask task, Schema inputSchema)
+    {
+        Schema outputSchema = ColumnFilterPlugin.buildOutputSchema(task, inputSchema);
+        return new JsonVisitor(task, inputSchema, outputSchema);
+    }
+
+    @Test
+    public void buildShouldVisitSet()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$.json1.a.a.a\"}",
+                "add_columns:",
+                "  - {name: \"$.json1.b.b[1].b\", type: string, default: foo}",
+                "drop_columns:",
+                "  - {name: \"$.json1.c.c[*].c\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        assertTrue(subject.shouldVisit("$.json1.a.a.a"));
+        assertTrue(subject.shouldVisit("$.json1.a.a"));
+        assertTrue(subject.shouldVisit("$.json1.a"));
+        assertTrue(subject.shouldVisit("$.json1.b.b[1].b"));
+        assertTrue(subject.shouldVisit("$.json1.b.b[1]"));
+        assertTrue(subject.shouldVisit("$.json1.b.b"));
+        assertTrue(subject.shouldVisit("$.json1.b"));
+        assertTrue(subject.shouldVisit("$.json1.c.c[*].c"));
+        assertTrue(subject.shouldVisit("$.json1.c.c[*]"));
+        assertTrue(subject.shouldVisit("$.json1.c.c"));
+        assertTrue(subject.shouldVisit("$.json1.c"));
+        assertTrue(subject.shouldVisit("$.json1"));
+        assertFalse(subject.shouldVisit("$.json2"));
+    }
+
+    @Test
+    public void buildJsonSchema_DropColumns()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: $.json1.a.default}",
+                "  - {name: $.json1.a.copy}",
+                "  - {name: \"$.json1.a.copy_array[1]\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        assertFalse(subject.jsonDropColumns.containsKey("$.json1"));
+        assertTrue(subject.jsonDropColumns.containsKey("$.json1.a"));
+        assertTrue(subject.jsonDropColumns.containsKey("$.json1.a.copy_array"));
+
+        {
+            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a");
+            assertEquals(2, jsonColumns.size());
+            assertTrue(jsonColumns.contains("$.json1.a.default"));
+            assertTrue(jsonColumns.contains("$.json1.a.copy"));
+        }
+
+        {
+            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a.copy_array");
+            assertEquals(1, jsonColumns.size());
+            assertTrue(jsonColumns.contains("$.json1.a.copy_array[1]"));
+        }
+    }
+
+    @Test
+    public void buildJsonSchema_AddColumns()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: $.json1.a.default, type: string, default: foo}",
+                "  - {name: $.json1.a.copy, src: $.json1.a.src}",
+                "  - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        assertFalse(subject.jsonAddColumns.containsKey("$.json1"));
+        assertTrue(subject.jsonAddColumns.containsKey("$.json1.a"));
+        assertTrue(subject.jsonAddColumns.containsKey("$.json1.a.copy_array"));
+
+        {
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a");
+            assertEquals(2, jsonColumns.size());
+            String[] keys = jsonColumns.keySet().toArray(new String[0]);
+            JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
+            assertEquals("$.json1.a.default", keys[0]);
+            assertEquals("$.json1.a.default", values[0].getPath());
+            assertEquals("$.json1.a.copy", keys[1]);
+            assertEquals("$.json1.a.copy", values[1].getPath());
+        }
+
+        {
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a.copy_array");
+            assertEquals(1, jsonColumns.size());
+            String[] keys = jsonColumns.keySet().toArray(new String[0]);
+            JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
+            assertEquals("$.json1.a.copy_array[1]", keys[0]);
+            assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
+        }
+    }
+
+    @Test
+    public void buildJsonSchema_Columns()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: $.json1.a.default, type: string, default: foo}",
+                "  - {name: $.json1.a.copy, src: $.json1.a.src}",
+                "  - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        assertFalse(subject.jsonColumns.containsKey("$.json1"));
+        assertTrue(subject.jsonColumns.containsKey("$.json1.a"));
+        assertTrue(subject.jsonColumns.containsKey("$.json1.a.copy_array"));
+
+        {
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a");
+            assertEquals(2, jsonColumns.size());
+            String[] keys = jsonColumns.keySet().toArray(new String[0]);
+            JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
+            assertEquals("$.json1.a.default", keys[0]);
+            assertEquals("$.json1.a.default", values[0].getPath());
+            assertEquals("$.json1.a.copy", keys[1]);
+            assertEquals("$.json1.a.copy", values[1].getPath());
+        }
+
+        {
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a.copy_array");
+            assertEquals(1, jsonColumns.size());
+            String[] keys = jsonColumns.keySet().toArray(new String[0]);
+            JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
+            assertEquals("$.json1.a.copy_array[1]", keys[0]);
+            assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
+        }
+    }
+
+    @Test
+    public void buildJsonSchema_Mix() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: $.json1.a.default}",
+                "add_columns:",
+                "  - {name: $.json1.a.copy, src: $.json1.a.src}",
+                "columns:",
+                "  - {name: \"$.json1.a.copy_array[1]\", src: \"$.json1.a.copy_array[0]\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        assertFalse(subject.jsonDropColumns.isEmpty());
+        assertFalse(subject.jsonAddColumns.isEmpty());
+        assertTrue(subject.jsonColumns.isEmpty()); // drop_columns overcome columns
+    }
+
+    @Test
+    public void visitMap_DropColumns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: $.json1.k1.k1}",
+                "  - {name: $.json1.k2}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":{}}", visited.toString());
+    }
+
+    @Test
+    public void visitMap_AddColumns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: $.json1.k3, type: json, default: \"{}\"}",
+                "  - {name: $.json1.k3.k3, type: string, default: v}",
+                "  - {name: $.json1.k4, src: $.json1.k2}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visitMap_Columns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: $.json1.k1}",
+                "  - {name: $.json1.k2.k2}", // $.json1.k2 must be specified now, or $.json.k2 will be removed entirely
+                "  - {name: $.json1.k3, type: json, default: \"{}\"}",
+                "  - {name: $.json1.k3.k3, type: string, default: v}",
+                "  - {name: $.json1.k4, src: $.json1.k2}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_DropColumns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: \"$.json1.k1[0].k1\"}",
+                "  - {name: \"$.json1.k2[*]\"}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_AddColumns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
+                "  - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
+                "  - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
+                "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"}],"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_Columns() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$.json1.k1\"}",
+                "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
+                "  - {name: \"$.json1.k2[0]\"}", // $.json1.k2 must be specified now, or $.json.k2 will be removed entirely
+                "  - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
+                "  - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
+                "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
+        Schema inputSchema = schema(
+                new Column(0, "json1", JSON),
+                new Column(1, "json2", JSON));
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-31 00:00:00.000000000 Z
+date: 2016-06-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -67,8 +67,9 @@ files:
 - src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
 - src/main/java/org/embulk/filter/column/JsonColumn.java
 - src/main/java/org/embulk/filter/column/JsonVisitor.java
-- src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.5.0.jar
+- src/test/java/org/embulk/filter/column/TestJsonColumn.java
+- src/test/java/org/embulk/filter/column/TestJsonVisitor.java
+- classpath/embulk-filter-column-0.5.1.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT

data/src/test/java/org/embulk/filter/TestColumnFilterPlugin.java

@@ -1,5 +0,0 @@
-package org.embulk.filter;
-
-public class TestColumnFilterPlugin
-{
-}