embulk-filter-column 0.5.4 → 0.6.0.pre1

data/src/main/java/org/embulk/filter/column/PathTokenUtil.java

@@ -0,0 +1,39 @@
+package org.embulk.filter.column;
+
+import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
+import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
+import com.dena.analytics.jsonpathcompiler.expressions.path.FunctionPathToken;
+import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
+import com.dena.analytics.jsonpathcompiler.expressions.path.PredicatePathToken;
+import com.dena.analytics.jsonpathcompiler.expressions.path.ScanPathToken;
+import org.embulk.config.ConfigException;
+
+public class PathTokenUtil
+{
+    public static void assertSupportedPathToken(PathToken pathToken, String path)
+    {
+        if (pathToken instanceof ArrayPathToken) {
+            ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
+            assertSupportedArrayPathToken(arrayIndexOperation, path);
+        }
+        else if (pathToken instanceof ScanPathToken) {
+            throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof FunctionPathToken) {
+            throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof PredicatePathToken) {
+            throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
+        }
+    }
+
+    public static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
+    {
+        if (arrayIndexOperation == null) {
+            throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
+        }
+        else if (!arrayIndexOperation.isSingleIndexOperation()) {
+            throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
+        }
+    }
+}

data/src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java

@@ -32,7 +32,7 @@ public class TestColumnFilterPlugin
     private ColumnFilterPlugin plugin;
 
     @Before
-    public void createReasource()
+    public void createResource()
     {
         plugin = new ColumnFilterPlugin();
     }

data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java

@@ -34,7 +34,7 @@ public class TestColumnVisitorImpl
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
     @Before
-    public void createReasource()
+    public void createResource()
     {
     }
 

data/src/test/java/org/embulk/filter/column/TestJsonColumn.java

@@ -15,7 +15,7 @@ public class TestJsonColumn
     {
         try {
             JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN);
-            assertEquals("$.foo.bar", column.getSrc());
+            assertEquals("$['foo']['bar']", column.getSrc());
             assertEquals(ValueFactory.newNil(), column.getDefaultValue());
         }
         catch (Exception e) {
@@ -24,8 +24,8 @@ public class TestJsonColumn
 
         try {
             Value defaultValue = ValueFactory.newBoolean(true);
-            JsonColumn column = new JsonColumn("$.foo.bar", Types.BOOLEAN, defaultValue);
-            assertEquals("$.foo.bar", column.getSrc());
+            JsonColumn column = new JsonColumn("$['foo']['bar']", Types.BOOLEAN, defaultValue);
+            assertEquals("$['foo']['bar']", column.getSrc());
             assertEquals(defaultValue, column.getDefaultValue());
         }
         catch (Exception e) {
@@ -36,22 +36,44 @@ public class TestJsonColumn
     @Test
     public void parentPath()
     {
-        assertEquals("$.foo.bar", JsonColumn.parentPath("$.foo.bar.baz"));
-        assertEquals("$.foo", JsonColumn.parentPath("$.foo.bar"));
-        assertEquals("$", JsonColumn.parentPath("$.foo"));
-        assertEquals("$.foo[0]", JsonColumn.parentPath("$.foo[0][1]"));
-        assertEquals("$.foo", JsonColumn.parentPath("$.foo[0]"));
+        assertEquals("$['foo']['bar']", JsonColumn.parentPath("$.foo.bar.baz"));
+        assertEquals("$['foo']", JsonColumn.parentPath("$.foo.bar"));
+        assertEquals("$", JsonColumn.parentPath("$['foo']"));
+        assertEquals("$['foo'][0]", JsonColumn.parentPath("$.foo[0][1]"));
+        assertEquals("$['foo']", JsonColumn.parentPath("$.foo[0]"));
         assertEquals("$", JsonColumn.parentPath("$[0]"));
     }
 
     @Test
-    public void baseName()
+    public void tailName()
     {
-        assertEquals("baz", JsonColumn.baseName("$.foo.bar.baz"));
-        assertEquals("bar", JsonColumn.baseName("$.foo.bar"));
-        assertEquals("foo", JsonColumn.baseName("$.foo"));
-        assertEquals("[1]", JsonColumn.baseName("$.foo[0][1]"));
-        assertEquals("[0]", JsonColumn.baseName("$.foo[0]"));
-        assertEquals("[0]", JsonColumn.baseName("$[0]"));
+        assertEquals("['baz']", JsonColumn.tailName("$['foo'].bar.baz"));
+        assertEquals("['bar']", JsonColumn.tailName("$.foo.bar"));
+        assertEquals("['foo']", JsonColumn.tailName("$.foo"));
+        assertEquals("[1]", JsonColumn.tailName("$.foo[0][1]"));
+        assertEquals("[0]", JsonColumn.tailName("$.foo[0]"));
+        assertEquals("[0]", JsonColumn.tailName("$[0]"));
     }
-}
+
+    @Test
+    public void getTailNameValue()
+    {
+        assertEquals(ValueFactory.newString("baz"), new JsonColumn("$['foo'].bar.baz", Types.BOOLEAN).getTailNameValue());
+        assertEquals(ValueFactory.newString("bar"), new JsonColumn("$.foo.bar", Types.BOOLEAN).getTailNameValue());
+        assertEquals(ValueFactory.newString("foo"), new JsonColumn("$.foo", Types.BOOLEAN).getTailNameValue());
+        assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0][1]", Types.BOOLEAN).getTailNameValue());
+        assertEquals(ValueFactory.newNil(), new JsonColumn("$.foo[0]", Types.BOOLEAN).getTailNameValue());
+        assertEquals(ValueFactory.newNil(), new JsonColumn("$[0]", Types.BOOLEAN).getTailNameValue());
+    }
+
+    @Test
+    public void tailIndex()
+    {
+        assertEquals(null, JsonColumn.tailIndex("$['foo'].bar.baz"));
+        assertEquals(null, JsonColumn.tailIndex("$.foo.bar"));
+        assertEquals(null, JsonColumn.tailIndex("$.foo"));
+        assertEquals(new Long(1), JsonColumn.tailIndex("$.foo[0][1]"));
+        assertEquals(new Long(0), JsonColumn.tailIndex("$.foo[0]"));
+        assertEquals(new Long(0), JsonColumn.tailIndex("$[0]"));
+    }
+}

data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java

@@ -13,6 +13,7 @@ import org.junit.Test;
 import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
+import com.dena.analytics.jsonpathcompiler.InvalidPathException;
 
 import static org.embulk.spi.type.Types.JSON;
 import static org.junit.Assert.assertEquals;
@@ -28,7 +29,7 @@ public class TestJsonVisitor
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
     @Before
-    public void createReasource()
+    public void createResource()
     {
     }
 
@@ -85,19 +86,19 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertTrue(subject.shouldVisit("$.json1.a.a.a"));
-        assertTrue(subject.shouldVisit("$.json1.a.a"));
-        assertTrue(subject.shouldVisit("$.json1.a"));
-        assertTrue(subject.shouldVisit("$.json1.b.b[1].b"));
-        assertTrue(subject.shouldVisit("$.json1.b.b[1]"));
-        assertTrue(subject.shouldVisit("$.json1.b.b"));
-        assertTrue(subject.shouldVisit("$.json1.b"));
-        assertTrue(subject.shouldVisit("$.json1.c.c[*].c"));
-        assertTrue(subject.shouldVisit("$.json1.c.c[*]"));
-        assertTrue(subject.shouldVisit("$.json1.c.c"));
-        assertTrue(subject.shouldVisit("$.json1.c"));
-        assertTrue(subject.shouldVisit("$.json1"));
-        assertFalse(subject.shouldVisit("$.json2"));
+        assertTrue(subject.shouldVisit("$['json1']['a']['a']['a']"));
+        assertTrue(subject.shouldVisit("$['json1']['a']['a']"));
+        assertTrue(subject.shouldVisit("$['json1']['a']"));
+        assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]['b']"));
+        assertTrue(subject.shouldVisit("$['json1']['b']['b'][1]"));
+        assertTrue(subject.shouldVisit("$['json1']['b']['b']"));
+        assertTrue(subject.shouldVisit("$['json1']['b']"));
+        assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]['c']"));
+        assertTrue(subject.shouldVisit("$['json1']['c']['c'][*]"));
+        assertTrue(subject.shouldVisit("$['json1']['c']['c']"));
+        assertTrue(subject.shouldVisit("$['json1']['c']"));
+        assertTrue(subject.shouldVisit("$['json1']"));
+        assertFalse(subject.shouldVisit("$['json2']"));
     }
 
     @Test
@@ -115,21 +116,21 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertFalse(subject.jsonDropColumns.containsKey("$.json1"));
-        assertTrue(subject.jsonDropColumns.containsKey("$.json1.a"));
-        assertTrue(subject.jsonDropColumns.containsKey("$.json1.a.copy_array"));
+        assertFalse(subject.jsonDropColumns.containsKey("$['json1']"));
+        assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']"));
+        assertTrue(subject.jsonDropColumns.containsKey("$['json1']['a']['copy_array']"));
 
         {
-            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a");
+            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']");
             assertEquals(2, jsonColumns.size());
-            assertTrue(jsonColumns.contains("$.json1.a.default"));
-            assertTrue(jsonColumns.contains("$.json1.a.copy"));
+            assertTrue(jsonColumns.contains("$['json1']['a']['default']"));
+            assertTrue(jsonColumns.contains("$['json1']['a']['copy']"));
         }
 
         {
-            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$.json1.a.copy_array");
+            HashSet<String> jsonColumns = subject.jsonDropColumns.get("$['json1']['a']['copy_array']");
             assertEquals(1, jsonColumns.size());
-            assertTrue(jsonColumns.contains("$.json1.a.copy_array[1]"));
+            assertTrue(jsonColumns.contains("$['json1']['a']['copy_array'][1]"));
         }
     }
 
@@ -148,28 +149,28 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertFalse(subject.jsonAddColumns.containsKey("$.json1"));
-        assertTrue(subject.jsonAddColumns.containsKey("$.json1.a"));
-        assertTrue(subject.jsonAddColumns.containsKey("$.json1.a.copy_array"));
+        assertFalse(subject.jsonAddColumns.containsKey("$['json1']"));
+        assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']"));
+        assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']['copy_array']"));
 
         {
-            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a");
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']");
             assertEquals(2, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
-            assertEquals("$.json1.a.default", keys[0]);
-            assertEquals("$.json1.a.default", values[0].getPath());
-            assertEquals("$.json1.a.copy", keys[1]);
-            assertEquals("$.json1.a.copy", values[1].getPath());
+            assertEquals("$['json1']['a']['default']", keys[0]);
+            assertEquals("$['json1']['a']['default']", values[0].getPath());
+            assertEquals("$['json1']['a']['copy']", keys[1]);
+            assertEquals("$['json1']['a']['copy']", values[1].getPath());
         }
 
         {
-            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$.json1.a.copy_array");
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']['copy_array']");
             assertEquals(1, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
-            assertEquals("$.json1.a.copy_array[1]", keys[0]);
-            assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
+            assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
+            assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
         }
     }
 
@@ -188,28 +189,28 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertFalse(subject.jsonColumns.containsKey("$.json1"));
-        assertTrue(subject.jsonColumns.containsKey("$.json1.a"));
-        assertTrue(subject.jsonColumns.containsKey("$.json1.a.copy_array"));
+        assertFalse(subject.jsonColumns.containsKey("$['json1']"));
+        assertTrue(subject.jsonColumns.containsKey("$['json1']['a']"));
+        assertTrue(subject.jsonColumns.containsKey("$['json1']['a']['copy_array']"));
 
         {
-            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a");
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']");
             assertEquals(2, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
-            assertEquals("$.json1.a.default", keys[0]);
-            assertEquals("$.json1.a.default", values[0].getPath());
-            assertEquals("$.json1.a.copy", keys[1]);
-            assertEquals("$.json1.a.copy", values[1].getPath());
+            assertEquals("$['json1']['a']['default']", keys[0]);
+            assertEquals("$['json1']['a']['default']", values[0].getPath());
+            assertEquals("$['json1']['a']['copy']", keys[1]);
+            assertEquals("$['json1']['a']['copy']", values[1].getPath());
         }
 
         {
-            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$.json1.a.copy_array");
+            HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']['copy_array']");
             assertEquals(1, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
-            assertEquals("$.json1.a.copy_array[1]", keys[0]);
-            assertEquals("$.json1.a.copy_array[1]", values[0].getPath());
+            assertEquals("$['json1']['a']['copy_array'][1]", keys[0]);
+            assertEquals("$['json1']['a']['copy_array'][1]", values[0].getPath());
         }
     }
 
@@ -257,7 +258,7 @@ public class TestJsonVisitor
                 k1, ValueFactory.newMap(k1, v),
                 k2, ValueFactory.newMap(k2, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":{}}", visited.toString());
     }
 
@@ -284,7 +285,7 @@ public class TestJsonVisitor
                 k1, ValueFactory.newMap(k1, v),
                 k2, ValueFactory.newMap(k2, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
     }
 
@@ -313,7 +314,7 @@ public class TestJsonVisitor
                 k1, ValueFactory.newMap(k1, v),
                 k2, ValueFactory.newMap(k2, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
     }
 
@@ -339,7 +340,7 @@ public class TestJsonVisitor
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
                 k2, ValueFactory.newArray(v, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
     }
 
@@ -367,7 +368,7 @@ public class TestJsonVisitor
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
                 k2, ValueFactory.newArray(v, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
     }
 
@@ -397,7 +398,394 @@ public class TestJsonVisitor
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
                 k2, ValueFactory.newArray(v, v));
 
-        MapValue visited = subject.visit("$.json1", map).asMapValue();
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
         assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
     }
-}
+
+    @Test
+    public void visitMap_dropColumnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: \"$['json1']['k1']['k1']\"}",
+                "  - {name: \"$['json1']['k2']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .add("json2", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":{}}", visited.toString());
+    }
+
+    @Test
+    public void visitMap_addColumnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
+                "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
+                "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .add("json2", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visitMap_columnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$['json1']['k1']\"}",
+                "  - {name: \"$['json1']['k2']['k2']\"}",
+                "  - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
+                "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
+                "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":{"k1":"v"},"k2":{"k1":"v","k2":"v"}}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newMap(k1, v),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_dropColumnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "drop_columns:",
+                "  - {name: \"$['json1']['k1'][0]['k1']\"}",
+                "  - {name: \"$['json1']['k2'][*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .add("json2", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_addColumnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
+                "  - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
+                "  - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
+                "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .add("json2", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"}],"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+    }
+
+    @Test
+    public void visitArray_columnsUsingBracketNotation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$['json1']['k1']\"}",
+                "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
+                "  - {name: \"$['json1']['k2'][0]\"}",
+                "  - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
+                "  - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
+                "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
+        Value k1 = ValueFactory.newString("k1");
+        Value k2 = ValueFactory.newString("k2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
+                k2, ValueFactory.newArray(v, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+    }
+
+    // Because the dot notation is converted to single quotes by default,
+    // it can be mixed with the bracket notation of single quotes
+    @Test
+    public void visit_withDotAndBracket()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                " - {name: \"$.json1['k_1']\"}",
+                " - {name: \"$.json1['k_1'][0]['k_1']\"}",
+                " - {name: \"$['json1']['k_2']\"}",
+                " - {name: \"$['json1']['k_2']['k_2']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
+        Value k1 = ValueFactory.newString("k_1");
+        Value k2 = ValueFactory.newString("k_2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
+    }
+
+    // Because the bracket notation of double quotes converted to single quotes internally
+    // it can be mixed with the bracket notation of single quotes
+    @Test
+    public void visit_withSingleQuotesAndDoubleQuotes()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                " - {name: \"$['json1']['k_1']\", src: \"$['json1']['k.1']\"}",
+                " - {name: '$[\"json1\"][\"k_1\"][0][\"k_1\"]', src: '$[\"json1\"][\"k_1\"][0][\"k.1\"]'}",
+                " - {name: '$[\"json1\"][\"k_2\"]', src: '$[\"json1\"][\"k.2\"]'}",
+                " - {name: '$[\"json1\"][\"k_2\"][\"k_2\"]', src: '$[\"json1\"][\"k_2\"][\"k.2\"]'}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
+        Value k1 = ValueFactory.newString("k.1");
+        Value k2 = ValueFactory.newString("k.2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k_1\":[{\"k_1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visit_withComplexRename()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                " - {name: \"$.json1['k____1']\", src: \"$.json1['k.-=+1']\"}",
+                " - {name: \"$.json1['k____1'][0]['k____1']\", src: \"$.json1['k____1'][0]['k.-=+1']\"}",
+                " - {name: \"$['json1']['k_2']\", src: \"$['json1']['k.2']\"}",
+                " - {name: \"$['json1']['k_2']['k_2']\", src: \"$['json1']['k_2']['k.2']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k.1":[{"k.1":"v"}], "k.2":{"k.2":"v"}}
+        Value k1 = ValueFactory.newString("k.-=+1");
+        Value k2 = ValueFactory.newString("k.2");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(
+                k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
+                k2, ValueFactory.newMap(k2, v));
+
+        MapValue visited = subject.visit("$['json1']", map).asMapValue();
+        assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
+    }
+
+    @Test
+    public void visit_withColumnNameIncludingSingleQuotes()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                " - {name: \"$[\\\"'json1\\\"]['k1']\"}");
+        Schema inputSchema = Schema.builder()
+                .add("'json1", JSON)
+                .build();
+        JsonVisitor subject = jsonVisitor(task, inputSchema);
+
+        // {"k1":"v"}
+        Value k1 = ValueFactory.newString("k1");
+        Value v = ValueFactory.newString("v");
+        Value map = ValueFactory.newMap(k1, v);
+
+        MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
+        assertEquals("{\"k1\":\"v\"}", visited.toString());
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithMultiProperties() {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$['json1','k1']\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    // It is recognized multi properties if the square brackets does not close properly
+    @Test(expected = InvalidPathException.class)
+    public void constructor_mustBeRaisedInvalidPathExceptionWithPropertyIsNotSeparatedByCommas()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                " - name: \"$['json1'}['k1']\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithFunctionPathToken()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$['json1'].length()\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithPredicatePathToken()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.store.book[?(@.price < 10)]\"");
+        Schema inputSchema = Schema.builder()
+                .add("store", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithScanPathToken()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.json1..key1\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.json1[0,1]\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperationAtMiddlePosition()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.json1[0,1].key1\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperation()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.json1[1:2]\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperationAtMiddlePosition()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "- name: \"$.json1[1:2].key1\"");
+        Schema inputSchema = Schema.builder()
+                .add("json1", JSON)
+                .build();
+        jsonVisitor(task, inputSchema);
+    }
+}