embulk-filter-column 0.6.0.pre5 → 0.6.0.pre6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 123d41cdcecc136bceb265c9d77f40719717955e
-  data.tar.gz: 250eef3285efb754089b9b6de2fe181721d960a5
+  metadata.gz: 28dd927b9569d7dc3b9f9fef47bf6c4d165f0910
+  data.tar.gz: 83e6e55aba543066e42ae4170d75dd081042cec6
 SHA512:
-  metadata.gz: 5e7a8e33c7c8bbe68701174b3533dd4d568324280ee74c35a1187aadcf4eff77ce0f74c790e5c4295d972cd59bea08063e8444d39336e86698e4f927b575371e
-  data.tar.gz: 5bd87c09fa8528e632171e67594a2326e2a00514e7083b57eb9d72a4c6f6ae23628ff14c01f877b9d6819c27d0f08f524690a4534dc2039a33f75379ae947347
+  metadata.gz: 22ab4bf44258f1719eb99114a2fa9e9e45c34ef11795769ecfc356bdd46dc8735632045470567823e3b1e5ec8d1ec1ce1149d08472246d1dbcfd3287ab3abfa8
+  data.tar.gz: 792770891fa30db0126b01aff7081076bff3bc129d223529180ad33fe90b0f9c34e0d3e142cf1898cccad96c2bfd44f56a6ff3e66c63f7c2bb8a5ab62b101093

data/README.md

@@ -145,10 +145,6 @@ I mean that below example does not work yet (`$.payload.foo` and `$.payload.bar`
 - {name: $.payload.foo.dest, src: $.payload.bar.src}
 ```
 
-## ToDo
-
-* Write test
-
 ## Development
 
 Run example:

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.6.0.pre5"
+version = "0.6.0.pre6"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/filter/column/ColumnFilterPlugin.java

@@ -2,14 +2,13 @@ package org.embulk.filter.column;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
-
 import org.embulk.spi.Column;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
@@ -21,14 +20,11 @@ import org.embulk.spi.Schema;
 import org.embulk.spi.SchemaConfigException;
 import org.embulk.spi.time.TimestampParser;
 import org.embulk.spi.type.Type;
-
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 
 import java.util.List;
 
-import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
-
 public class ColumnFilterPlugin implements FilterPlugin
 {
     private static final Logger logger = Exec.getLogger(ColumnFilterPlugin.class);
@@ -95,11 +91,7 @@ public class ColumnFilterPlugin implements FilterPlugin
 
     private void configure(PluginTask task)
     {
-        List<ColumnConfig> columns = task.getColumns();
-        List<ColumnConfig> addColumns = task.getAddColumns();
-        List<ColumnConfig> dropColumns = task.getDropColumns();
-
-        if (columns.size() > 0 && dropColumns.size() > 0) {
+        if (task.getColumns().size() > 0 && task.getDropColumns().size() > 0) {
             throw new ConfigException("Either of \"columns\", \"drop_columns\" can be specified.");
         }
     }

data/src/main/java/org/embulk/filter/column/ColumnVisitorImpl.java

@@ -1,8 +1,9 @@
 package org.embulk.filter.column;
 
-import io.github.medjed.jsonpathcompiler.expressions.Utils;
 import com.google.common.base.Throwables;
 
+import io.github.medjed.jsonpathcompiler.expressions.Utils;
+
 import org.embulk.filter.column.ColumnFilterPlugin.ColumnConfig;
 import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
 
@@ -126,7 +127,7 @@ public class ColumnVisitorImpl implements ColumnVisitor
         }
         else if (type instanceof LongType) {
             if (columnConfig.getDefault().isPresent()) {
-                return new Long(columnConfig.getDefault().get().toString());
+                return Long.valueOf(columnConfig.getDefault().get().toString());
             }
         }
         else if (type instanceof DoubleType) {

data/src/main/java/org/embulk/filter/column/JsonColumn.java

@@ -5,8 +5,8 @@ import io.github.medjed.jsonpathcompiler.expressions.path.ArrayIndexOperation;
 import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
-import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.RootPathToken;
 import io.github.medjed.jsonpathcompiler.expressions.path.WildcardPathToken;
 import org.embulk.config.ConfigException;
 import org.embulk.spi.type.Type;
@@ -33,7 +33,7 @@ public class JsonColumn
     private StringValue srcParentPathValue = null;
     private Value srcTailNameValue = null;
 
-    public static int WILDCARD_INDEX = -1;
+    public static final int WILDCARD_INDEX = -1;
 
     public JsonColumn(String path, Type type)
     {
@@ -101,7 +101,7 @@ public class JsonColumn
             return arrayIndexOperation.indexes().get(0).longValue();
         }
         else if (tail instanceof WildcardPathToken) {
-            return new Long(WILDCARD_INDEX);
+            return Long.valueOf(WILDCARD_INDEX);
         }
         else {
             return null;

data/src/main/java/org/embulk/filter/column/JsonPathUtil.java

@@ -14,6 +14,8 @@ import org.embulk.config.ConfigException;
 
 public class JsonPathUtil
 {
+    private JsonPathUtil() {}
+
     public static void assertJsonPathFormat(String path)
     {
         Path compiledPath;

data/src/main/java/org/embulk/filter/column/JsonVisitor.java

@@ -337,19 +337,26 @@ public class JsonVisitor
         }
         else if (this.jsonColumns.containsKey(rootPath)) {
             for (JsonColumn jsonColumn : this.jsonColumns.get(rootPath).values()) {
-                int src = jsonColumn.getSrcTailIndex().intValue();
-                if (src == JsonColumn.WILDCARD_INDEX) {
-                    for (int i = 0; i < size; i++) {
-                        Value v = arrayValue.get(i);
-                        if (v == null) {
-                            v = jsonColumn.getDefaultValue();
+                int i = jsonColumn.getTailIndex().intValue();
+                if (i == JsonColumn.WILDCARD_INDEX) {
+                    for (i = 0; i < size; i++) {
+                        int src = jsonColumn.getSrcTailIndex().intValue();
+                        if (src == JsonColumn.WILDCARD_INDEX) {
+                            Value v = arrayValue.get(i);
+                            if (v == null) {
+                                v = jsonColumn.getDefaultValue();
+                            }
+                            String newPath = jsonColumn.getPath(); // == newArrayJsonPath(rootPath, i); // [*]
+                            Value visited = visit(newPath, v);
+                            newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
+                        }
+                        else {
+                            assert (false); // not supported yet
                         }
-                        String newPath = jsonColumn.getPath();
-                        Value visited = visit(newPath, v);
-                        newValue.add(j++, visited == null ? ValueFactory.newNil() : visited);
                     }
                 }
                 else {
+                    int src = jsonColumn.getSrcTailIndex().intValue();
                     Value v = (src < arrayValue.size() ? arrayValue.get(src) : null);
                     if (v == null) {
                         v = jsonColumn.getDefaultValue();

data/src/test/java/org/embulk/filter/column/TestColumnVisitorImpl.java

@@ -100,8 +100,8 @@ public class TestColumnVisitorImpl
                 .add("remove_me", STRING)
                 .build();
         List<Object[]> records = filter(task, inputSchema,
-                Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
-                Timestamp.ofEpochSecond(0), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me");
+                Timestamp.ofEpochSecond(0), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "remove_me",
+                Timestamp.ofEpochSecond(0), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "remove_me");
 
         assertEquals(2, records.size());
 
@@ -111,9 +111,9 @@ public class TestColumnVisitorImpl
             assertEquals(6, record.length);
             assertEquals(Timestamp.ofEpochSecond(0), record[0]);
             assertEquals("string", record[1]);
-            assertEquals(new Boolean(true), record[2]);
-            assertEquals(new Long(0), record[3]);
-            assertEquals(new Double(0.5), record[4]);
+            assertEquals(Boolean.valueOf(true), record[2]);
+            assertEquals(Long.valueOf(0), record[3]);
+            assertEquals(Double.valueOf(0.5), record[4]);
             assertEquals(ValueFactory.newString("json"), record[5]);
         }
     }
@@ -140,7 +140,7 @@ public class TestColumnVisitorImpl
                 .add("remove_me", STRING)
                 .build();
         List<Object[]> records = filter(task, inputSchema,
-                Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "remove_me",
+                Timestamp.ofEpochSecond(1436745600), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "remove_me",
                 null, null, null, null, null, null, "remove_me");
 
         assertEquals(2, records.size());
@@ -151,9 +151,9 @@ public class TestColumnVisitorImpl
             assertEquals(6, record.length);
             assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
             assertEquals("string", record[1]);
-            assertEquals(new Boolean(true), record[2]);
-            assertEquals(new Long(0), record[3]);
-            assertEquals(new Double(0.5), record[4]);
+            assertEquals(Boolean.valueOf(true), record[2]);
+            assertEquals(Long.valueOf(0), record[3]);
+            assertEquals(Double.valueOf(0.5), record[4]);
             assertEquals(ValueFactory.newString("json"), record[5]);
         }
         {
@@ -161,9 +161,9 @@ public class TestColumnVisitorImpl
             assertEquals(6, record.length);
             assertEquals(Timestamp.ofEpochSecond(1436745600), record[0]);
             assertEquals("string", record[1]);
-            assertEquals(new Boolean(true), record[2]);
-            assertEquals(new Long(0), record[3]);
-            assertEquals(new Double(0.5), record[4]);
+            assertEquals(Boolean.valueOf(true), record[2]);
+            assertEquals(Long.valueOf(0), record[3]);
+            assertEquals(Double.valueOf(0.5), record[4]);
             assertEquals("{\"foo\":\"bar\"}", record[5].toString());
         }
     }
@@ -213,7 +213,7 @@ public class TestColumnVisitorImpl
                 .add("keep_me", STRING)
                 .build();
         List<Object[]> records = filter(task, inputSchema,
-                Timestamp.ofEpochSecond(1436745600), "string", new Boolean(true), new Long(0), new Double(0.5), ValueFactory.newString("json"), "keep_me",
+                Timestamp.ofEpochSecond(1436745600), "string", Boolean.valueOf(true), Long.valueOf(0), Double.valueOf(0.5), ValueFactory.newString("json"), "keep_me",
                 null, null, null, null, null, null, "keep_me");
 
         assertEquals(2, records.size());
@@ -259,9 +259,9 @@ public class TestColumnVisitorImpl
             assertEquals("keep_me", record[0]);
             assertEquals(Timestamp.ofEpochSecond(1436745600), record[1]);
             assertEquals("string", record[2]);
-            assertEquals(new Boolean(true), record[3]);
-            assertEquals(new Long(0), record[4]);
-            assertEquals(new Double(0.5), record[5]);
+            assertEquals(Boolean.valueOf(true), record[3]);
+            assertEquals(Long.valueOf(0), record[4]);
+            assertEquals(Double.valueOf(0.5), record[5]);
             assertEquals("{\"foo\":\"bar\"}", record[6].toString());
         }
     }

data/src/test/java/org/embulk/filter/column/TestJsonColumn.java

@@ -72,8 +72,8 @@ public class TestJsonColumn
         assertEquals(null, JsonColumn.getTailIndex("$['foo'].bar.baz"));
         assertEquals(null, JsonColumn.getTailIndex("$.foo.bar"));
         assertEquals(null, JsonColumn.getTailIndex("$.foo"));
-        assertEquals(new Long(1), JsonColumn.getTailIndex("$.foo[0][1]"));
-        assertEquals(new Long(0), JsonColumn.getTailIndex("$.foo[0]"));
-        assertEquals(new Long(0), JsonColumn.getTailIndex("$[0]"));
+        assertEquals(Long.valueOf(1), JsonColumn.getTailIndex("$.foo[0][1]"));
+        assertEquals(Long.valueOf(0), JsonColumn.getTailIndex("$.foo[0]"));
+        assertEquals(Long.valueOf(0), JsonColumn.getTailIndex("$[0]"));
     }
 }

data/src/test/java/org/embulk/filter/column/TestJsonVisitor.java

@@ -1,7 +1,6 @@
 package org.embulk.filter.column;
 
 import org.embulk.config.ConfigException;
-import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
 import org.embulk.filter.column.ColumnFilterPlugin.PluginTask;
@@ -21,14 +20,13 @@ import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
 
 public class TestJsonVisitor
 {
     @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    public org.embulk.EmbulkTestRuntime runtime = new org.embulk.EmbulkTestRuntime();
 
     @Before
     public void createResource()
@@ -153,24 +151,24 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void configException_Columns()
+    public void assertDoNotEndsWithArrayWildcard_AddColumns()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
-                "columns:",
-                "  - {name: \"$.json1.b.b[*]\"}");
+                "add_columns:",
+                "  - {name: \"$.json1.b.b[*]\", type: json, default: []}");
         Schema inputSchema = Schema.builder().build();
         // b[*] should be written as b
         jsonVisitor(task, inputSchema);
     }
 
     @Test(expected = ConfigException.class)
-    public void buildJsonAddColumns_ConfigException()
+    public void assertDoNotEndsWithArrayWildcard_Columns()
     {
         PluginTask task = taskFromYamlString(
                 "type: column",
-                "add_columns:",
-                "  - {name: \"$.json1.b.b[*]\", type: json, default: []}");
+                "columns:",
+                "  - {name: \"$.json1.b.b[*]\"}");
         Schema inputSchema = Schema.builder().build();
         // b[*] should be written as b
         jsonVisitor(task, inputSchema);
@@ -219,18 +217,6 @@ public class TestJsonVisitor
         }
     }
 
-    @Test(expected = ConfigException.class)
-    public void buildJsonColumns_ConfigException()
-    {
-        PluginTask task = taskFromYamlString(
-                "type: column",
-                "columns:",
-                "  - {name: \"$.json1.b.b[*]\"}");
-        Schema inputSchema = Schema.builder().build();
-        // b[*] should be written as b
-        jsonVisitor(task, inputSchema);
-    }
-
     @Test
     public void buildJsonColumns()
     {
@@ -445,6 +431,7 @@ public class TestJsonVisitor
                 "  - {name: \"$.json1.k1[1]\", src: \"$.json1.k1[0]\"}",
                 "  - {name: \"$.json1.k2[0]\"}",
                 "  - {name: \"$.json1.k3[*].k1\"}",
+                "  - {name: \"$.json1.k3[*].k3\", src: \"$.json1.k3[*].k1\"}",
                 "  - {name: \"$.json1.k4[*].k1\", type: string, default: v}",
                 "  - {name: \"$.json1.k5[0].k1\", type: string, default: v}");
         Schema inputSchema = Schema.builder()
@@ -464,7 +451,7 @@ public class TestJsonVisitor
                 k3, ValueFactory.newArray(ValueFactory.newMap(k1, v, k2, v)));
 
         MapValue visited = subject.visit("$['json1']", map).asMapValue();
-        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k1\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
+        assertEquals("{\"k1\":[{\"k1\":\"v\"}],\"k2\":[\"v\"],\"k3\":[{\"k1\":\"v\",\"k3\":\"v\"}],\"k4\":[],\"k5\":[{\"k1\":\"v\"}]}", visited.toString());
     }
 
     @Test
@@ -731,7 +718,8 @@ public class TestJsonVisitor
     }
 
     @Test(expected = ConfigException.class)
-    public void configException_MultiProperties() {
+    public void configException_MultiProperties()
+    {
         PluginTask task = taskFromYamlString(
                 "type: column",
                 "columns:",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.6.0.pre5
+  version: 0.6.0.pre6
 platform: ruby
 authors:
 - Naotoshi Seo
@@ -11,33 +11,33 @@ cert_chain: []
 date: 2016-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: A filter plugin for Embulk to filter out columns.
 email:
 - sonots@gmail.com
@@ -82,7 +82,7 @@ files:
 - classpath/accessors-smart-1.1.jar
 - classpath/asm-5.0.3.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-filter-column-0.6.0.pre5.jar
+- classpath/embulk-filter-column-0.6.0.pre6.jar
 - classpath/json-smart-2.2.1.jar
 - classpath/JsonPathCompiler-0.0.12.jar
 - classpath/slf4j-api-1.7.21.jar