embulk-filter-column 0.6.0.pre2 → 0.6.0.pre3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b693b9f514309a94b2fea4e85714e6e0c9de0f1
-  data.tar.gz: ed56dc5b557605436985615eacdbc71b700a7640
+  metadata.gz: d26c69b6a7bfb4a6c4d68c6967301a0dc32562f5
+  data.tar.gz: c674640721b35540a9aba9aef1ad76c5cb5adc96
 SHA512:
-  metadata.gz: 5d1399f4e807811687c032a1738254bc2013d0a760a88343ec03d3bd3b36309cadb46f1c805859d68724a1bb40b0bc73e1908dbab939cb54e453f7a6db93b5b1
-  data.tar.gz: 7937b001df59b540c8a10fcbd4e3c707565d5e9f9ca4e919d96b7403252d74b654ee8ca0b476fe53d504192ef9302b678f209bed30071e873af48f0a74d8ae2e
+  metadata.gz: 9cc5e8165d85afed48b65302d0baa9f54e35b1ae5438f390b5b5970d8342e14cf72b30f137ca901229578c3493b23a4c3a6286ea076501bc926eaed97491f70f
+  data.tar.gz: 867fa85522091768287f5b4ade0b7e17947899e1f6ccbeeb017d5115fb0468d418c62712af308a80580afe1be6bb5a2f3ac73c7c1274b11d4c674dd8c8635a66

data/CHANGELOG.md

@@ -1,8 +1,9 @@
-# 0.6.0 (2016-10-01)
+# 0.6.0 (2016-XX-XX)
 
 Enhancements: 
 
 * Support bracket notation in json path (thanks to @kysnm)
+* Autocomplete ancestor json paths
 
 # 0.5.4 (2016-08-05)
 

data/README.md

@@ -133,27 +133,15 @@ NOTE:
 
 NOTE:
 
-To deeply visit json path such as `$.payload.foo.bar`, you have to write its upper paths together like:
+Rename or copy of json paths by `src` option is only partially supported yet. The parent json path must be same like:
 
 ```
-- (name: $.payload.foo}
-- {name: $.payload.foo.bar}
-```
-
-NOTE:
-
-`src` (to rename or copy columns) is only partially supported yet. The upper json path must be same like:
-
-```
-- {name: $.payload.foo}
 - {name: $.payload.foo.dest, src: $.payload.foo.src}
 ```
 
-Below does not work yet.
+I mean that below example does not work yet (`$.payload.foo` and `$.payload.bar`)
 
 ```
-- {name: $.payload.foo}
-- {name: $.payload.bar}
 - {name: $.payload.foo.dest, src: $.payload.bar.src}
 ```
 

data/build.gradle

@@ -15,14 +15,14 @@ configurations {
     provided
 }
 
-version = "0.6.0.pre2"
+version = "0.6.0.pre3"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
-    compile  'com.dena.analytics:JsonPathCompiler:0.0.6'
+    compile  "io.github.medjed:JsonPathCompiler:0.0.+"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.+:tests"

data/example/add_columns.txt

@@ -0,0 +1,11 @@
+2016-10-20 02:29:31.933 +0900: Embulk v0.8.6
+2016-10-20 02:29:32.859 +0900 [INFO] (0001:preview): Loaded plugin embulk/filter/column from a load path
+2016-10-20 02:29:32.876 +0900 [INFO] (0001:preview): Listing local files at directory 'example' filtering filename by prefix 'example.csv'
+2016-10-20 02:29:32.882 +0900 [INFO] (0001:preview): Loading files [example/example.csv]
++-------------------------+---------+---------------------------------------------+--------------+-------------------------------------------------------------+----------+-------------------------+-------------------------+-------------------+
+|          time:timestamp | id:long |                                 name:string | score:double |                                                   json:json | foo:long |             d:timestamp |             t:timestamp | copy_score:double |
++-------------------------+---------+---------------------------------------------+--------------+-------------------------------------------------------------+----------+-------------------------+-------------------------+-------------------+
+| 2015-07-13 00:00:00 UTC |       0 | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |       1370.0 | {"foo":"foo","bar":"bar","d":"2015-07-13","copy_foo":"foo"} |        1 | 2015-07-12 15:00:00 UTC | 2015-07-13 00:00:00 UTC |            1370.0 |
+| 2015-07-13 00:00:00 UTC |       0 | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |       1370.0 |   {"foo":"foo2","bar":1,"d":"2015-07-13","copy_foo":"foo2"} |        1 | 2015-07-12 15:00:00 UTC | 2015-07-13 00:00:00 UTC |            1370.0 |
+|                         |         |                                             |       9170.0 |                                                             |        1 | 2015-07-12 15:00:00 UTC | 2015-07-13 00:00:00 UTC |            9170.0 |
++-------------------------+---------+---------------------------------------------+--------------+-------------------------------------------------------------+----------+-------------------------+-------------------------+-------------------+

data/example/add_columns.yml

@@ -23,7 +23,7 @@ filters:
       - {name: d, type: timestamp, default: "2015-07-13"}
       - {name: t, type: timestamp, default: "2015-07-13 00:00:00", timezone: "UTC", format: "%Y-%m-%d %H:%M:%S"}
       - {name: copy_score, src: score}
-      - {name: $.json.foo, type: long, default: 1}
+      - {name: $.json.bar, type: long, default: 1}
       - {name: $.json.d, type: string, default: "2015-07-13"}
       - {name: $.json.copy_foo, src: $.json.foo}
 out:

data/example/bracket_notations.txt

@@ -0,0 +1,11 @@
+2016-10-20 02:29:47.367 +0900: Embulk v0.8.6
+2016-10-20 02:29:48.254 +0900 [INFO] (0001:preview): Loaded plugin embulk/filter/column from a load path
+2016-10-20 02:29:48.270 +0900 [INFO] (0001:preview): Listing local files at directory 'example' filtering filename by prefix 'example.csv'
+2016-10-20 02:29:48.274 +0900 [INFO] (0001:preview): Loading files [example/example.csv]
++-------------------------+---------------------------------------------+----------+---------+-------------------+---------------------------------------------------------------------------------------------------------+
+|          time:timestamp |                                 name:string | foo:long | id:long | copy_score:double |                                                                                               json:json |
++-------------------------+---------------------------------------------+----------+---------+-------------------+---------------------------------------------------------------------------------------------------------+
+| 2015-07-13 00:00:00 UTC | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |        1 |       0 |            1370.0 |                   {"foo":"foo","copy_foo":"foo","array_a":[{"a":"a0"}],"array_b":[{"b":"b"},{"b":"b"}]} |
+| 2015-07-13 00:00:00 UTC | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |        1 |       0 |            1370.0 |                 {"foo":"foo2","copy_foo":"foo2","array_a":[{"a":"a0"}],"array_b":[{"b":"b"},{"b":"b"}]} |
+| 2015-07-13 00:00:00 UTC |                                         foo |        1 |         |            9170.0 | {"foo":"FOO","copy_foo":"FOO","array_a":[{"a":"default0"}],"array_b":[{"b":"default"},{"b":"default"}]} |
++-------------------------+---------------------------------------------+----------+---------+-------------------+---------------------------------------------------------------------------------------------------------+

data/example/columns.txt

@@ -0,0 +1,11 @@
+2016-10-20 02:30:03.618 +0900: Embulk v0.8.6
+2016-10-20 02:30:04.499 +0900 [INFO] (0001:preview): Loaded plugin embulk/filter/column from a load path
+2016-10-20 02:30:04.516 +0900 [INFO] (0001:preview): Listing local files at directory 'example' filtering filename by prefix 'example.csv'
+2016-10-20 02:30:04.521 +0900 [INFO] (0001:preview): Loading files [example/example.csv]
++-------------------------+---------------------------------------------+----------+---------+-------------------+----------------------------------+
+|          time:timestamp |                                 name:string | foo:long | id:long | copy_score:double |                        json:json |
++-------------------------+---------------------------------------------+----------+---------+-------------------+----------------------------------+
+| 2015-07-13 00:00:00 UTC | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |        1 |       0 |            1370.0 |   {"foo":"foo","copy_foo":"foo"} |
+| 2015-07-13 00:00:00 UTC | Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |        1 |       0 |            1370.0 | {"foo":"foo2","copy_foo":"foo2"} |
+| 2015-07-13 00:00:00 UTC |                                         foo |        1 |         |            9170.0 |   {"foo":"FOO","copy_foo":"FOO"} |
++-------------------------+---------------------------------------------+----------+---------+-------------------+----------------------------------+

data/example/drop_columns.txt

@@ -0,0 +1,11 @@
+2016-10-20 02:30:16.407 +0900: Embulk v0.8.6
+2016-10-20 02:30:17.290 +0900 [INFO] (0001:preview): Loaded plugin embulk/filter/column from a load path
+2016-10-20 02:30:17.305 +0900 [INFO] (0001:preview): Listing local files at directory 'example' filtering filename by prefix 'example.csv'
+2016-10-20 02:30:17.310 +0900 [INFO] (0001:preview): Loading files [example/example.csv]
++---------------------------------------------+--------------+---------------+
+|                                 name:string | score:double |     json:json |
++---------------------------------------------+--------------+---------------+
+| Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |       1370.0 | {"bar":"bar"} |
+| Vqjht6YEUBsMPXmoW1iOGFROZF27pBzz0TUkOKeDXEY |       1370.0 |            {} |
+|                                             |       9170.0 |               |
++---------------------------------------------+--------------+---------------+

data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java

@@ -27,7 +27,7 @@ import org.slf4j.Logger;
 
 import java.util.List;
 
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 
 public class ColumnFilterPlugin implements FilterPlugin
 {

data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java

@@ -1,6 +1,6 @@
 package org.embulk.filter.column;
 
-import com.dena.analytics.jsonpathcompiler.expressions.Utils;
+import io.github.medjed.jsonpathcompiler.expressions.Utils;
 import com.google.common.base.Throwables;
 
 import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;

data/src/main/java/org/embulk/filter/column/JsonColumn.java

@@ -1,12 +1,12 @@
 package org.embulk.filter.column;
 
-import com.dena.analytics.jsonpathcompiler.expressions.Path;
-import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
-import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.RootPathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PropertyPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayIndexOperation;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
 import org.embulk.config.ConfigException;
 import org.embulk.spi.type.Type;
 import org.msgpack.value.StringValue;
@@ -56,17 +56,14 @@ public class JsonColumn
         this.pathValue = ValueFactory.newString(path);
         this.parentPath = compiledPath.getParentPath();
 
-        if (compiledRoot.getTailPath().equals("[*]")) {
-            throw new ConfigException(String.format("%s wrongly ends with [*], perhaps you can remove the [*]", path));
-        }
-        this.tailIndex = tailIndex(compiledRoot);
+        this.tailIndex = getTailIndex(compiledRoot);
         this.parentPathValue = ValueFactory.newString(parentPath);
         String tailName = getTailName(compiledRoot);
         this.tailNameValue = tailName == null ? ValueFactory.newNil() : ValueFactory.newString(tailName);
 
         this.srcValue = ValueFactory.newString(this.src);
         this.srcParentPath = compiledSrc.getParentPath();
-        this.srcTailIndex = tailIndex(compiledSrcRoot);
+        this.srcTailIndex = getTailIndex(compiledSrcRoot);
         this.srcParentPathValue = ValueFactory.newString(this.srcParentPath);
         String srcTailName = getTailName(compiledSrcRoot);
         this.srcTailNameValue = srcTailName == null ? ValueFactory.newNil() : ValueFactory.newString(srcTailName);
@@ -92,12 +89,12 @@ public class JsonColumn
         }
     }
 
-    private Long tailIndex(RootPathToken root)
+    private Long getTailIndex(RootPathToken root)
     {
         PathToken tail = root.getTail();
         if (tail instanceof ArrayPathToken) {
             ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) tail).getArrayIndexOperation();
-            PathTokenUtil.assertSupportedArrayPathToken(arrayIndexOperation, path);
+            JsonPathUtil.assertSupportedArrayPathToken(arrayIndexOperation, path);
             return arrayIndexOperation.indexes().get(0).longValue();
         }
         else {
@@ -135,7 +132,7 @@ public class JsonColumn
         return parentPath;
     }
 
-    public Long tailIndex()
+    public Long getTailIndex()
     {
         return tailIndex;
     }
@@ -186,7 +183,7 @@ public class JsonColumn
         return ((RootPathToken) PathCompiler.compile(path).getRoot()).getTailPath();
     }
 
-    public static Long tailIndex(String path)
+    public static Long getTailIndex(String path)
     {
         Path compiledPath = PathCompiler.compile(path);
         PathToken tail = ((RootPathToken) compiledPath.getRoot()).getTail();

data/src/main/java/org/embulk/filter/column/JsonPathUtil.java

@@ -0,0 +1,76 @@
+package org.embulk.filter.column;
+
+import io.github.medjed.jsonpathcompiler.InvalidPathException;
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayIndexOperation;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.FunctionPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PredicatePathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.ScanPathToken;
+import org.embulk.config.ConfigException;
+
+public class JsonPathUtil
+{
+    public static void assertJsonPathFormat(String path)
+    {
+        Path compiledPath;
+        try {
+            compiledPath = PathCompiler.compile(path);
+        }
+        catch (InvalidPathException e) {
+            throw new ConfigException(String.format("jsonpath %s, %s", path, e.getMessage()));
+        }
+        PathToken pathToken = compiledPath.getRoot();
+        while (true) {
+            assertSupportedPathToken(pathToken, path);
+            if (pathToken.isLeaf()) {
+                break;
+            }
+            pathToken = pathToken.next();
+        }
+    }
+
+    public static void assertSupportedPathToken(PathToken pathToken, String path)
+    {
+        if (pathToken instanceof ArrayPathToken) {
+            ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
+            assertSupportedArrayPathToken(arrayIndexOperation, path);
+        }
+        else if (pathToken instanceof ScanPathToken) {
+            throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof FunctionPathToken) {
+            throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof PredicatePathToken) {
+            throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
+        }
+    }
+
+    public static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
+    {
+        if (arrayIndexOperation == null) {
+            throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
+        }
+        else if (!arrayIndexOperation.isSingleIndexOperation()) {
+            throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
+        }
+    }
+
+    public static void assertDoNotEndsWithArrayWildcard(String path)
+    {
+        Path compiledPath;
+        try {
+            compiledPath = PathCompiler.compile(path);
+        }
+        catch (InvalidPathException e) {
+            throw new ConfigException(String.format("jsonpath %s, %s", path, e.getMessage()));
+        }
+        if (((RootPathToken) compiledPath.getRoot()).getTailPath().equals("[*]")) {
+            throw new ConfigException(String.format("%s wrongly ends with [*], perhaps you can remove the [*]", compiledPath.toString()));
+        }
+    }
+}

data/src/main/java/org/embulk/filter/column/JsonVisitor.java

@@ -1,9 +1,10 @@
 package org.embulk.filter.column;
 
-import com.dena.analytics.jsonpathcompiler.InvalidPathException;
-import com.dena.analytics.jsonpathcompiler.expressions.Path;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathCompiler;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.InvalidPathException;
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
 import org.embulk.config.ConfigException;
 import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
 import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
@@ -18,6 +19,7 @@ import org.embulk.spi.type.LongType;
 import org.embulk.spi.type.StringType;
 import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
 import org.msgpack.value.ArrayValue;
 import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
@@ -38,9 +40,13 @@ public class JsonVisitor
     final PluginTask task;
     final Schema inputSchema;
     final Schema outputSchema;
+    // jsonpath
     final HashSet<String> shouldVisitSet = new HashSet<>();
+    // parent jsonpath => { jsonpath => json column }
     final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonColumns = new HashMap<>();
+    // parent jsonpath => { jsonpath => json column }
     final HashMap<String, LinkedHashMap<String, JsonColumn>> jsonAddColumns = new HashMap<>();
+    // parent jsonpath => [ jsonpath ]
     final HashMap<String, HashSet<String>> jsonDropColumns = new HashMap<>();
 
     JsonVisitor(PluginTask task, Schema inputSchema, Schema outputSchema)
@@ -92,6 +98,18 @@ public class JsonVisitor
         jsonColumns.get(parentPath).put(compiledPath.toString(), value);
     }
 
+    private boolean jsonColumnsContainsKey(String path)
+    {
+        Path compiledPath = PathCompiler.compile(path);
+        String parentPath = compiledPath.getParentPath();
+        if (jsonColumns.containsKey(parentPath)) {
+            return jsonColumns.get(parentPath).containsKey(compiledPath.toString());
+        }
+        else {
+            return false;
+        }
+    }
+
     private void jsonAddColumnsPut(String path, JsonColumn value)
     {
         Path compiledPath = PathCompiler.compile(path);
@@ -102,6 +120,18 @@ public class JsonVisitor
         jsonAddColumns.get(parentPath).put(compiledPath.toString(), value);
     }
 
+    private boolean jsonAddColumnsContainsKey(String path)
+    {
+        Path compiledPath = PathCompiler.compile(path);
+        String parentPath = compiledPath.getParentPath();
+        if (jsonAddColumns.containsKey(parentPath)) {
+            return jsonAddColumns.get(parentPath).containsKey(compiledPath.toString());
+        }
+        else {
+            return false;
+        }
+    }
+
     private void jsonDropColumnsPut(String path)
     {
         Path compiledPath = PathCompiler.compile(path);
@@ -112,68 +142,98 @@ public class JsonVisitor
         jsonDropColumns.get(parentPath).add(compiledPath.toString());
     }
 
-    // build jsonColumns, jsonAddColumns, and jsonDropColumns
-    private void buildJsonSchema()
+    private void buildJsonColumns()
     {
         List<ColumnConfig> columns = task.getColumns();
-        List<ColumnConfig> addColumns = task.getAddColumns();
-        List<ColumnConfig> dropColumns = task.getDropColumns();
-
-        int i = 0;
-        if (dropColumns.size() > 0) {
-            for (ColumnConfig dropColumn : dropColumns) {
-                String name = dropColumn.getName();
-                // skip NON json path notation to build output schema
-                if (! PathCompiler.isProbablyJsonPath(name)) {
-                    continue;
+        for (ColumnConfig column : columns) {
+            String name = column.getName();
+            // skip NON json path notation to build output schema
+            if (! PathCompiler.isProbablyJsonPath(name)) {
+                continue;
+            }
+            JsonPathUtil.assertDoNotEndsWithArrayWildcard(name);
+            // automatically fill ancestor jsonpaths
+            for (JsonColumn ancestorJsonColumn : getAncestorJsonColumnList(name)) {
+                String ancestorJsonPath = ancestorJsonColumn.getPath();
+                if (!jsonColumnsContainsKey(ancestorJsonPath)) {
+                    jsonColumnsPut(ancestorJsonPath, ancestorJsonColumn);
                 }
-                jsonDropColumnsPut(name);
+            }
+            // leaf jsonpath
+            if (column.getSrc().isPresent()) {
+                String src = column.getSrc().get();
+                jsonColumnsPut(name, new JsonColumn(name, null, null, src));
+            }
+            else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
+                Type type = column.getType().get();
+                Value defaultValue = getDefault(task, name, type, column);
+                jsonColumnsPut(name, new JsonColumn(name, type, defaultValue));
+            }
+            else {
+                Type type = column.getType().isPresent() ? column.getType().get() : null;
+                jsonColumnsPut(name, new JsonColumn(name, type));
             }
         }
-        else if (columns.size() > 0) {
-            for (ColumnConfig column : columns) {
-                String name = column.getName();
-                // skip NON json path notation to build output schema
-                if (! PathCompiler.isProbablyJsonPath(name)) {
-                    continue;
-                }
-                if (column.getSrc().isPresent()) {
-                    String src = column.getSrc().get();
-                    jsonColumnsPut(name, new JsonColumn(name, null, null, src));
-                }
-                else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
-                    Type type = column.getType().get();
-                    Value defaultValue = getDefault(task, name, type, column);
-                    jsonColumnsPut(name, new JsonColumn(name, type, defaultValue));
-                }
-                else {
-                    Type type = column.getType().isPresent() ? column.getType().get() : null;
-                    jsonColumnsPut(name, new JsonColumn(name, type));
+    }
+
+    private void buildJsonAddColumns()
+    {
+        List<ColumnConfig> addColumns = task.getAddColumns();
+        for (ColumnConfig column : addColumns) {
+            String name = column.getName();
+            // skip NON json path notation to build output schema
+            if (! PathCompiler.isProbablyJsonPath(name)) {
+                continue;
+            }
+            JsonPathUtil.assertDoNotEndsWithArrayWildcard(name);
+            // automatically fill ancestor jsonpaths
+            for (JsonColumn ancestorJsonColumn : getAncestorJsonColumnList(name)) {
+                String ancestorJsonPath = ancestorJsonColumn.getPath();
+                if (!jsonAddColumnsContainsKey(ancestorJsonPath)) {
+                    jsonAddColumnsPut(ancestorJsonPath, ancestorJsonColumn);
                 }
             }
+            // leaf jsonpath
+            if (column.getSrc().isPresent()) {
+                String src = column.getSrc().get();
+                jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
+            }
+            else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
+                Type type = column.getType().get();
+                Value defaultValue = getDefault(task, name, type, column);
+                jsonAddColumnsPut(name, new JsonColumn(name, type, defaultValue));
+            }
+            else {
+                throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"src\", or \"type\" and \"default\"", name));
+            }
         }
+    }
 
-        // Add columns to last. If you want to add to head or middle, you can use `columns` option
-        if (addColumns.size() > 0) {
-            for (ColumnConfig column : addColumns) {
-                String name = column.getName();
-                // skip NON json path notation to build output schema
-                if (! PathCompiler.isProbablyJsonPath(name)) {
-                    continue;
-                }
-                if (column.getSrc().isPresent()) {
-                    String src = column.getSrc().get();
-                    jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
-                }
-                else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
-                    Type type = column.getType().get();
-                    Value defaultValue = getDefault(task, name, type, column);
-                    jsonAddColumnsPut(name, new JsonColumn(name, type, defaultValue));
-                }
-                else {
-                    throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"src\", or \"type\" and \"default\"", name));
-                }
+    private void buildJsonDropColumns()
+    {
+        List<ColumnConfig> dropColumns = task.getDropColumns();
+        for (ColumnConfig dropColumn : dropColumns) {
+            String name = dropColumn.getName();
+            // skip NON json path notation to build output schema
+            if (! PathCompiler.isProbablyJsonPath(name)) {
+                continue;
             }
+            jsonDropColumnsPut(name);
+        }
+    }
+
+    // build jsonColumns, jsonAddColumns, and jsonDropColumns
+    private void buildJsonSchema()
+    {
+        if (task.getDropColumns().size() > 0) {
+            buildJsonDropColumns();
+        }
+        else if (task.getColumns().size() > 0) {
+            buildJsonColumns();
+        }
+        // Add columns to last. If you want to add to head or middle, you can use `columns` option
+        if (task.getAddColumns().size() > 0) {
+            buildJsonAddColumns();
         }
     }
 
@@ -189,23 +249,53 @@ public class JsonVisitor
             if (!PathCompiler.isProbablyJsonPath(name)) {
                 continue;
             }
-            Path path;
-            try {
-                path = PathCompiler.compile(name);
-            } catch (InvalidPathException e) {
-                throw new ConfigException(String.format("path %s, %s", name, e.getMessage()));
+            JsonPathUtil.assertJsonPathFormat(name);
+            for (JsonColumn ancestorJsonColumn : getAncestorJsonColumnList(name)) {
+                this.shouldVisitSet.add(ancestorJsonColumn.getPath());
             }
-            PathToken parts = path.getRoot();
-            int count = parts.getTokenCount();
-            StringBuilder partialPath = new StringBuilder("$");
-            // skip "$"
-            for (int i = 1; i < count; i++) {
-                parts = parts.next();
-                PathTokenUtil.assertSupportedPathToken(parts, name);
-                partialPath.append(parts.getPathFragment().toString());
-                this.shouldVisitSet.add(partialPath.toString());
+            Path path = PathCompiler.compile(name);
+            this.shouldVisitSet.add(path.toString());
+        }
+    }
+
+    /*
+     * <pre>
+     * $['foo']['bar'][0]['baz']
+     * #=>
+     * name: $['foo'], type: json, default: {}
+     * name: $['foo']['bar'], type: json, default: []
+     * name: $['foo']['bar'][0], type: json, default: {}
+     * </pre>
+     *
+     * @return ancestors as an array
+     */
+    public static ArrayList<JsonColumn> getAncestorJsonColumnList(String path)
+    {
+        ArrayList<JsonColumn> ancestorJsonColumnList = new ArrayList<>();
+        Path compiledPath;
+        try {
+            compiledPath = PathCompiler.compile(path);
+        }
+        catch (InvalidPathException e) {
+            throw new ConfigException(String.format("jsonpath %s, %s", path, e.getMessage()));
+        }
+        StringBuilder partialPath = new StringBuilder("$");
+        PathToken parts = compiledPath.getRoot();
+        parts = parts.next(); // skip "$"
+        while (! parts.isLeaf()) {
+            partialPath.append(parts.getPathFragment());
+            PathToken next = parts.next();
+            JsonColumn jsonColumn;
+            if (next instanceof ArrayPathToken) {
+                jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newArray(new Value[0], false));
+            }
+            else {
+                jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newMap(new Value[0]));
             }
+            ancestorJsonColumnList.add(jsonColumn);
+            parts = next;
         }
+        return ancestorJsonColumnList;
     }
 
     boolean shouldVisit(String jsonPath)
@@ -253,7 +343,7 @@ public class JsonVisitor
                 }
                 String newPath = jsonColumn.getPath();
                 Value visited = visit(newPath, v);
-                // int i = jsonColumn.tailIndex().intValue();
+                // int i = jsonColumn.getTailIndex().intValue();
                 // index is shifted, so j++ is used.
                 newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
             }
@@ -267,6 +357,12 @@ public class JsonVisitor
         }
         if (this.jsonAddColumns.containsKey(rootPath)) {
             for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
+                int i = jsonColumn.getTailIndex().intValue();
+                if (i < size) {
+                    // index for add_columns must be larger than size
+                    // just skip because we can not raise ConfigException beforehand for flexible JSON
+                    continue;
+                }
                 int src = jsonColumn.getSrcTailIndex().intValue();
                 Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
                 if (v == null) {
@@ -326,6 +422,12 @@ public class JsonVisitor
         if (this.jsonAddColumns.containsKey(rootPath)) {
             Map<Value, Value> map = mapValue.map();
             for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
+                Value k = jsonColumn.getTailNameValue();
+                if (map.containsKey(k)) {
+                    // key must be different with already existing one for add_columns
+                    // just skip because we can not raise ConfigException beforehand for flexible JSON
+                    continue;
+                }
                 Value src = jsonColumn.getSrcTailNameValue();
                 Value v = map.get(src);
                 if (v == null) {

data/src/test/java/org/embulk/filter/column/TestJsonColumn.java

@@ -67,13 +67,13 @@ public class TestJsonColumn
     }
 
     @Test
-    public void tailIndex()
+    public void getTailIndex()
     {
-        assertEquals(null, JsonColumn.tailIndex("$['foo'].bar.baz"));
-        assertEquals(null, JsonColumn.tailIndex("$.foo.bar"));
-        assertEquals(null, JsonColumn.tailIndex("$.foo"));
-        assertEquals(new Long(1), JsonColumn.tailIndex("$.foo[0][1]"));
-        assertEquals(new Long(0), JsonColumn.tailIndex("$.foo[0]"));
-        assertEquals(new Long(0), JsonColumn.tailIndex("$[0]"));
+        assertEquals(null, JsonColumn.getTailIndex("$['foo'].bar.baz"));
+        assertEquals(null, JsonColumn.getTailIndex("$.foo.bar"));
+        assertEquals(null, JsonColumn.getTailIndex("$.foo"));
+        assertEquals(new Long(1), JsonColumn.getTailIndex("$.foo[0][1]"));
+        assertEquals(new Long(0), JsonColumn.getTailIndex("$.foo[0]"));
+        assertEquals(new Long(0), JsonColumn.getTailIndex("$[0]"));
     }
-}
+}

data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java

@@ -14,13 +14,14 @@ import org.junit.rules.ExpectedException;
 import org.msgpack.value.MapValue;
 import org.msgpack.value.Value;
 import org.msgpack.value.ValueFactory;
-import com.dena.analytics.jsonpathcompiler.InvalidPathException;
 
 import static org.embulk.spi.type.Types.JSON;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 
@@ -58,16 +59,32 @@ public class TestJsonVisitor
         return new JsonVisitor(task, inputSchema, outputSchema);
     }
 
-    @Test(expected = ConfigException.class)
-    public void configException_Columns()
+    @Test
+    public void getAncestorJsonColumnList()
     {
-        PluginTask task = taskFromYamlString(
-                "type: column",
-                "columns:",
-                "  - {name: \"$.json1.b.b[*]\"}");
-        Schema inputSchema = Schema.builder().build();
-        // b[*] should be written as b
-        jsonVisitor(task, inputSchema);
+        ArrayList<JsonColumn> subject;
+
+        subject = JsonVisitor.getAncestorJsonColumnList("$.json1.a.default");
+        assertEquals("$['json1']", subject.get(0).getPath());
+        assertTrue(subject.get(0).getDefaultValue().isMapValue());
+        assertEquals("$['json1']['a']", subject.get(1).getPath());
+        assertTrue(subject.get(1).getDefaultValue().isMapValue());
+
+        subject = JsonVisitor.getAncestorJsonColumnList("$.json1.a[0].default");
+        assertEquals("$['json1']", subject.get(0).getPath());
+        assertTrue(subject.get(0).getDefaultValue().isMapValue());
+        assertEquals("$['json1']['a']", subject.get(1).getPath());
+        assertTrue(subject.get(1).getDefaultValue().isArrayValue());
+        assertEquals("$['json1']['a'][0]", subject.get(2).getPath());
+        assertTrue(subject.get(2).getDefaultValue().isMapValue());
+
+        subject = JsonVisitor.getAncestorJsonColumnList("$.json1.a.default[0]");
+        assertEquals("$['json1']", subject.get(0).getPath());
+        assertTrue(subject.get(0).getDefaultValue().isMapValue());
+        assertEquals("$['json1']['a']", subject.get(1).getPath());
+        assertTrue(subject.get(1).getDefaultValue().isMapValue());
+        assertEquals("$['json1']['a']['default']", subject.get(2).getPath());
+        assertTrue(subject.get(2).getDefaultValue().isArrayValue());
     }
 
     @Test
@@ -103,7 +120,7 @@ public class TestJsonVisitor
     }
 
     @Test
-    public void buildJsonSchema_DropColumns()
+    public void buildJsonDropColumns()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -135,8 +152,32 @@ public class TestJsonVisitor
         }
     }
 
+    @Test(expected = ConfigException.class)
+    public void configException_Columns()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$.json1.b.b[*]\"}");
+        Schema inputSchema = Schema.builder().build();
+        // b[*] should be written as b
+        jsonVisitor(task, inputSchema);
+    }
+
+    @Test(expected = ConfigException.class)
+    public void buildJsonAddColumns_ConfigException()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "add_columns:",
+                "  - {name: \"$.json1.b.b[*]\", type: json, default: []}");
+        Schema inputSchema = Schema.builder().build();
+        // b[*] should be written as b
+        jsonVisitor(task, inputSchema);
+    }
+
     @Test
-    public void buildJsonSchema_AddColumns()
+    public void buildJsonAddColumns()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -150,19 +191,22 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertFalse(subject.jsonAddColumns.containsKey("$['json1']"));
+        assertTrue(subject.jsonAddColumns.containsKey("$"));
+        assertTrue(subject.jsonAddColumns.containsKey("$['json1']"));
         assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']"));
         assertTrue(subject.jsonAddColumns.containsKey("$['json1']['a']['copy_array']"));
 
         {
             HashMap<String, JsonColumn> jsonColumns = subject.jsonAddColumns.get("$['json1']['a']");
-            assertEquals(2, jsonColumns.size());
+            assertEquals(3, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
             assertEquals("$['json1']['a']['default']", keys[0]);
             assertEquals("$['json1']['a']['default']", values[0].getPath());
             assertEquals("$['json1']['a']['copy']", keys[1]);
             assertEquals("$['json1']['a']['copy']", values[1].getPath());
+            assertEquals("$['json1']['a']['copy_array']", keys[2]);
+            assertEquals("$['json1']['a']['copy_array']", values[2].getPath());
         }
 
         {
@@ -175,8 +219,20 @@ public class TestJsonVisitor
         }
     }
 
+    @Test(expected = ConfigException.class)
+    public void buildJsonColumns_ConfigException()
+    {
+        PluginTask task = taskFromYamlString(
+                "type: column",
+                "columns:",
+                "  - {name: \"$.json1.b.b[*]\"}");
+        Schema inputSchema = Schema.builder().build();
+        // b[*] should be written as b
+        jsonVisitor(task, inputSchema);
+    }
+
     @Test
-    public void buildJsonSchema_Columns()
+    public void buildJsonColumns()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -190,19 +246,23 @@ public class TestJsonVisitor
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        assertFalse(subject.jsonColumns.containsKey("$['json1']"));
+        // 1st level keys are parents of jsonpath
+        assertTrue(subject.jsonColumns.containsKey("$"));
+        assertTrue(subject.jsonColumns.containsKey("$['json1']"));
         assertTrue(subject.jsonColumns.containsKey("$['json1']['a']"));
         assertTrue(subject.jsonColumns.containsKey("$['json1']['a']['copy_array']"));
 
         {
             HashMap<String, JsonColumn> jsonColumns = subject.jsonColumns.get("$['json1']['a']");
-            assertEquals(2, jsonColumns.size());
+            assertEquals(3, jsonColumns.size());
             String[] keys = jsonColumns.keySet().toArray(new String[0]);
             JsonColumn[] values = jsonColumns.values().toArray(new JsonColumn[0]);
             assertEquals("$['json1']['a']['default']", keys[0]);
             assertEquals("$['json1']['a']['default']", values[0].getPath());
             assertEquals("$['json1']['a']['copy']", keys[1]);
             assertEquals("$['json1']['a']['copy']", values[1].getPath());
+            assertEquals("$['json1']['a']['copy_array']", keys[2]);
+            assertEquals("$['json1']['a']['copy_array']", values[2].getPath());
         }
 
         {
@@ -216,7 +276,7 @@ public class TestJsonVisitor
     }
 
     @Test
-    public void buildJsonSchema_Mix()
+    public void buildJsonSchema()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -269,7 +329,6 @@ public class TestJsonVisitor
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "add_columns:",
-                "  - {name: $.json1.k3, type: json, default: \"{}\"}",
                 "  - {name: $.json1.k3.k3, type: string, default: v}",
                 "  - {name: $.json1.k4, src: $.json1.k2}");
         Schema inputSchema = Schema.builder()
@@ -297,8 +356,7 @@ public class TestJsonVisitor
                 "type: column",
                 "columns:",
                 "  - {name: $.json1.k1}",
-                "  - {name: $.json1.k2.k2}", // $.json1.k2 must be specified now, or $.json.k2 will be removed entirely
-                "  - {name: $.json1.k3, type: json, default: \"{}\"}",
+                "  - {name: $.json1.k2.k2}",
                 "  - {name: $.json1.k3.k3, type: string, default: v}",
                 "  - {name: $.json1.k4, src: $.json1.k2}");
         Schema inputSchema = Schema.builder()
@@ -316,7 +374,7 @@ public class TestJsonVisitor
                 k2, ValueFactory.newMap(k2, v));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
     }
 
     @Test
@@ -352,8 +410,6 @@ public class TestJsonVisitor
                 "type: column",
                 "add_columns:",
                 "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
-                "  - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
-                "  - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
                 "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
@@ -379,11 +435,8 @@ public class TestJsonVisitor
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "columns:",
-                "  - {name: \"$.json1.k1\"}",
                 "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
-                "  - {name: \"$.json1.k2[0]\"}", // $.json1.k2 must be specified now, or $.json.k2 will be removed entirely
-                "  - {name: \"$.json1.k3\", type: json, default: \"[]\"}",
-                "  - {name: \"$.json1.k3[0]\", type: json, default: \"{}\"}",
+                "  - {name: \"$.json1.k2[0]\"}",
                 "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
@@ -400,7 +453,7 @@ public class TestJsonVisitor
                 k2, ValueFactory.newArray(v, v));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
     }
 
     @Test
@@ -435,7 +488,6 @@ public class TestJsonVisitor
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "add_columns:",
-                "  - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
                 "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
                 "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
         Schema inputSchema = Schema.builder()
@@ -464,7 +516,6 @@ public class TestJsonVisitor
                 "columns:",
                 "  - {name: \"$['json1']['k1']\"}",
                 "  - {name: \"$['json1']['k2']['k2']\"}",
-                "  - {name: \"$['json1']['k3']\", type: json, default: \"{}\"}",
                 "  - {name: \"$['json1']['k3']['k3']\", type: string, default: v}",
                 "  - {name: \"$['json1']['k4']\", src: \"$['json1']['k2']\"}");
         Schema inputSchema = Schema.builder()
@@ -481,7 +532,7 @@ public class TestJsonVisitor
                 k2, ValueFactory.newMap(k2, v));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
+        assertEquals("{\"k1\":{\"k1\":\"v\"},\"k2\":{\"k2\":\"v\"},\"k3\":{\"k3\":\"v\"},\"k4\":{\"k2\":\"v\"}}", visited.toString());
     }
 
     @Test
@@ -517,8 +568,6 @@ public class TestJsonVisitor
                 "type: column",
                 "add_columns:",
                 "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
-                "  - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
-                "  - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
                 "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
@@ -544,11 +593,8 @@ public class TestJsonVisitor
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "columns:",
-                "  - {name: \"$['json1']['k1']\"}",
                 "  - {name: \"$['json1']['k1'][1]\", src: \"$['json1']['k1'][0]\"}",
                 "  - {name: \"$['json1']['k2'][0]\"}",
-                "  - {name: \"$['json1']['k3']\", type: json, default: \"[]\"}",
-                "  - {name: \"$['json1']['k3'][0]\", type: json, default: \"{}\"}",
                 "  - {name: \"$['json1']['k3'][0]['k3']\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
@@ -564,7 +610,7 @@ public class TestJsonVisitor
                 k2, ValueFactory.newArray(v, v));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
     }
 
     // Because the dot notation is converted to single quotes by default,
@@ -652,13 +698,14 @@ public class TestJsonVisitor
         assertEquals("{\"k____1\":[{\"k____1\":\"v\"}],\"k_2\":{\"k_2\":\"v\"}}", visited.toString());
     }
 
+    /*
     @Test
     public void visit_withColumnNameIncludingSingleQuotes()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "columns:",
-                " - {name: \"$[\\\"'json1\\\"]['k1']\"}");
+                " - {name: \"$['\\\\'json1']['k1']\"}");
         Schema inputSchema = Schema.builder()
                 .add("'json1", JSON)
                 .build();
@@ -672,9 +719,10 @@ public class TestJsonVisitor
         MapValue visited = subject.visit("$['\\'json1']", map).asMapValue();
         assertEquals("{\"k1\":\"v\"}", visited.toString());
     }
+    */
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithMultiProperties() {
+    public void configException_MultiProperties() {
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "columns:",
@@ -687,7 +735,7 @@ public class TestJsonVisitor
 
     // It is recognized multi properties if the square brackets does not close properly
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedInvalidPathExceptionWithPropertyIsNotSeparatedByCommas()
+    public void configException_PropertyIsNotSeparatedByCommas()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -700,7 +748,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithFunctionPathToken()
+    public void configException_FunctionPathToken()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -713,7 +761,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithPredicatePathToken()
+    public void configException_PredicatePathToken()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -726,7 +774,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithScanPathToken()
+    public void configException_ScanPathToken()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -739,7 +787,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperation()
+    public void configException_MultiIndexOperation()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -752,7 +800,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithMultiIndexOperationAtMiddlePosition()
+    public void configException_IndexOperationAtMiddlePosition()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -765,7 +813,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperation()
+    public void configException_ArraySliceOperation()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -778,7 +826,7 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void constructor_mustBeRaisedConfigExceptionWithMArraySliceOperationAtMiddlePosition()
+    public void configException_MArraySliceOperationAtMiddlePosition()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -794,7 +842,7 @@ public class TestJsonVisitor
     public ExpectedException thrown = ExpectedException.none();
 
     @Test
-    public void constructor_mustBeRaisedConfigExceptionEffectively()
+    public void configException_PathCompileError()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
@@ -808,4 +856,4 @@ public class TestJsonVisitor
 
         jsonVisitor(task, inputSchema);
     }
-}
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.6.0.pre2
+  version: 0.6.0.pre3
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-17 00:00:00.000000000 Z
+date: 2016-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,9 +52,13 @@ files:
 - README.md
 - build.gradle
 - config/checkstyle/checkstyle.xml
+- example/add_columns.txt
 - example/add_columns.yml
+- example/bracket_notations.txt
 - example/bracket_notations.yml
+- example/columns.txt
 - example/columns.yml
+- example/drop_columns.txt
 - example/drop_columns.yml
 - example/edgecase.tsv
 - example/edgecase.yml
@@ -69,8 +73,8 @@ files:
 - src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java
 - src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java
 - src/main/java/org/embulk/filter/column/JsonColumn.java
+- src/main/java/org/embulk/filter/column/JsonPathUtil.java
 - src/main/java/org/embulk/filter/column/JsonVisitor.java
-- src/main/java/org/embulk/filter/column/PathTokenUtil.java
 - src/test/java/org/embulk/filter/column/TestColumnFilterPlugin.java
 - src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java
 - src/test/java/org/embulk/filter/column/TestJsonColumn.java
@@ -78,9 +82,9 @@ files:
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-filter-column-0.6.0.pre2.jar
+- classpath/embulk-filter-column-0.6.0.pre3.jar
 - classpath/json-smart-2.2.1.jar
-- classpath/JsonPathCompiler-0.0.6.jar
+- classpath/JsonPathCompiler-0.0.11.jar
 - classpath/slf4j-api-1.7.21.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:

data/src/main/java/org/embulk/filter/column/PathTokenUtil.java

@@ -1,39 +0,0 @@
-package org.embulk.filter.column;
-
-import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayIndexOperation;
-import com.dena.analytics.jsonpathcompiler.expressions.path.ArrayPathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.FunctionPathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.PredicatePathToken;
-import com.dena.analytics.jsonpathcompiler.expressions.path.ScanPathToken;
-import org.embulk.config.ConfigException;
-
-public class PathTokenUtil
-{
-    public static void assertSupportedPathToken(PathToken pathToken, String path)
-    {
-        if (pathToken instanceof ArrayPathToken) {
-            ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
-            assertSupportedArrayPathToken(arrayIndexOperation, path);
-        }
-        else if (pathToken instanceof ScanPathToken) {
-            throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
-        }
-        else if (pathToken instanceof FunctionPathToken) {
-            throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
-        }
-        else if (pathToken instanceof PredicatePathToken) {
-            throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
-        }
-    }
-
-    public static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
-    {
-        if (arrayIndexOperation == null) {
-            throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
-        }
-        else if (!arrayIndexOperation.isSingleIndexOperation()) {
-            throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
-        }
-    }
-}