embulk-filter-column 0.6.0.pre4 → 0.6.0.pre5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdfb9c7f62fa8351885416d57e2d517025af0111
-  data.tar.gz: e09a8286941062cbf5d47bf73e4fd75b011b062b
+  metadata.gz: 123d41cdcecc136bceb265c9d77f40719717955e
+  data.tar.gz: 250eef3285efb754089b9b6de2fe181721d960a5
 SHA512:
-  metadata.gz: 798f800dffdd91ba5333e96f8d9efaf22c448220c56e34f16029dd75b88035447b7acf60bb3312c2e30f1910292e0572db3b11e4b3247fb117e162d6bb417305
-  data.tar.gz: b6e459926af443198e02a716b20bc866ecfefa109b7771a70a2a238dd51a6d0b168745a9612c944bc6588c86caff8515bff61bda21e824ae00ec29838b0686ec
+  metadata.gz: 5e7a8e33c7c8bbe68701174b3533dd4d568324280ee74c35a1187aadcf4eff77ce0f74c790e5c4295d972cd59bea08063e8444d39336e86698e4f927b575371e
+  data.tar.gz: 5bd87c09fa8528e632171e67594a2326e2a00514e7083b57eb9d72a4c6f6ae23628ff14c01f877b9d6819c27d0f08f524690a4534dc2039a33f75379ae947347

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.6.0.pre4"
+version = "0.6.0.pre5"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/column/JsonColumn.java

@@ -7,6 +7,7 @@ import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.WildcardPathToken;
 import org.embulk.config.ConfigException;
 import org.embulk.spi.type.Type;
 import org.msgpack.value.StringValue;
@@ -32,6 +33,8 @@ public class JsonColumn
     private StringValue srcParentPathValue = null;
     private Value srcTailNameValue = null;
 
+    public static int WILDCARD_INDEX = -1;
+
     public JsonColumn(String path, Type type)
     {
         this(path, type, null, null);
@@ -97,6 +100,9 @@ public class JsonColumn
             JsonPathUtil.assertSupportedArrayPathToken(arrayIndexOperation, path);
             return arrayIndexOperation.indexes().get(0).longValue();
         }
+        else if (tail instanceof WildcardPathToken) {
+            return new Long(WILDCARD_INDEX);
+        }
         else {
             return null;
         }

data/src/main/java/org/embulk/filter/column/JsonVisitor.java

@@ -5,6 +5,7 @@ import io.github.medjed.jsonpathcompiler.expressions.Path;
 import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.WildcardPathToken;
 import org.embulk.config.ConfigException;
 import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
 import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
@@ -286,7 +287,7 @@ public class JsonVisitor
             partialPath.append(parts.getPathFragment());
             PathToken next = parts.next();
             JsonColumn jsonColumn;
-            if (next instanceof ArrayPathToken) {
+            if (next instanceof ArrayPathToken || next instanceof WildcardPathToken) {
                 jsonColumn = new JsonColumn(partialPath.toString(), Types.JSON, ValueFactory.newArray(new Value[0], false));
             }
             else {
@@ -337,15 +338,26 @@ public class JsonVisitor
         else if (this.jsonColumns.containsKey(rootPath)) {
             for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
                 int src = jsonColumn.getSrcTailIndex().intValue();
-                Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
-                if (v == null) {
-                    v = jsonColumn.getDefaultValue();
+                if (src == JsonColumn.WILDCARD_INDEX) {
+                    for (int i = 0; i < size; i++) {
+                        Value v = arrayValue.get(i);
+                        if (v == null) {
+                            v = jsonColumn.getDefaultValue();
+                        }
+                        String newPath = jsonColumn.getPath();
+                        Value visited = visit(newPath, v);
+                        newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
+                    }
+                }
+                else {
+                    Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
+                    if (v == null) {
+                        v = jsonColumn.getDefaultValue();
+                    }
+                    String newPath = jsonColumn.getPath();
+                    Value visited = visit(newPath, v);
+                    newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
                 }
-                String newPath = jsonColumn.getPath();
-                Value visited = visit(newPath, v);
-                // int i = jsonColumn.getTailIndex().intValue();
-                // index is shifted, so j++ is used.
-                newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
             }
         }
         else {
@@ -358,7 +370,7 @@ public class JsonVisitor
         if (this.jsonAddColumns.containsKey(rootPath)) {
             for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
                 int i = jsonColumn.getTailIndex().intValue();
-                if (i < size) {
+                if (i == JsonColumn.WILDCARD_INDEX || i < size) {
                     // index for add_columns must be larger than size
                     // just skip because we can not raise ConfigException beforehand for flexible JSON
                     continue;

data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java

@@ -384,23 +384,26 @@ public class TestJsonVisitor
                 "type: column",
                 "drop_columns:",
                 "  - {name: \"$.json1.k1[0].k1\"}",
-                "  - {name: \"$.json1.k2[*]\"}"); // ending with [*] is allowed for drop_columns, but not for others
+                "  - {name: \"$.json1.k2[*]\"}", // ending with [*] is allowed for drop_columns, but not for others
+                "  - {name: \"$.json1.k3[*].k1\"}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
                 .add("json2", JSON)
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        // {"k1":[{"k1":"v"}[,"k2":["v","v"]}
+        // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k3":"v"}]}
         Value k1 = ValueFactory.newString("k1");
         Value k2 = ValueFactory.newString("k2");
+        Value k3 = ValueFactory.newString("k3");
         Value v = ValueFactory.newString("v");
         Value map = ValueFactory.newMap(
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
-                k2, ValueFactory.newArray(v, v));
+                k2, ValueFactory.newArray(v, v),
+                k3, ValueFactory.newArray(ValueFactory.newMap(k1, v)));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{}],\"k2\":[]}", visited.toString());
+        assertEquals("{\"k1\":[{}],\"k2\":[],\"k3\":[{}]}", visited.toString());
     }
 
     @Test
@@ -410,23 +413,27 @@ public class TestJsonVisitor
                 "type: column",
                 "add_columns:",
                 "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
-                "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
+                "  - {name: \"$.json1.k3[*].k2\", type: string, default: v}",
+                "  - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
+                "  - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
                 .add("json2", JSON)
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        // {"k1":[{"k1":"v"}],"k2":["v","v"]}
+        // {"k1":[{"k1":"v"}],"k2":["v","v"],"k3":[{"k1":"v"}]}
         Value k1 = ValueFactory.newString("k1");
         Value k2 = ValueFactory.newString("k2");
+        Value k3 = ValueFactory.newString("k3");
         Value v = ValueFactory.newString("v");
         Value map = ValueFactory.newMap(
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v)),
-                k2, ValueFactory.newArray(v, v));
+                k2, ValueFactory.newArray(v, v),
+                k3, ValueFactory.newArray(ValueFactory.newMap(k1, v)));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+        assertEquals("{\"k1\":[{\"k1\":\"v\"},{\"k1\":\"v\"}],\"k2\":[\"v\",\"v\"],\"k3\":[{\"k1\":\"v\",\"k2\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
     }
 
     @Test
@@ -437,23 +444,27 @@ public class TestJsonVisitor
                 "columns:",
                 "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
                 "  - {name: \"$.json1.k2[0]\"}",
-                "  - {name: \"$.json1.k3[0].k3\", type: string, default: v}");
+                "  - {name: \"$.json1.k3[*].k1\"}",
+                "  - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
+                "  - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
                 .add("json1", JSON)
                 .add("json2", JSON)
                 .build();
         JsonVisitor subject = jsonVisitor(task, inputSchema);
 
-        // {"k1":[{"k1":"v"},"v"],"k2":["v","v"]}
+        // {"k1":[{"k1":"v"},"v"],"k2":["v","v"],"k3":[{"k1":"v","k2":"v"}]}
         Value k1 = ValueFactory.newString("k1");
         Value k2 = ValueFactory.newString("k2");
+        Value k3 = ValueFactory.newString("k3");
         Value v = ValueFactory.newString("v");
         Value map = ValueFactory.newMap(
                 k1, ValueFactory.newArray(ValueFactory.newMap(k1, v), v),
-                k2, ValueFactory.newArray(v, v));
+                k2, ValueFactory.newArray(v, v),
+                k3, ValueFactory.newArray(ValueFactory.newMap(k1, v, k2, v)));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k3\":\"v\"}]}", visited.toString());
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k1\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
     }
 
     @Test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.6.0.pre4
+  version: 0.6.0.pre5
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -82,7 +82,7 @@ files:
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-filter-column-0.6.0.pre4.jar
+- classpath/embulk-filter-column-0.6.0.pre5.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/JsonPathCompiler-0.0.12.jar
 - classpath/slf4j-api-1.7.21.jar