embulk-filter-column 0.5.0.pre1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1e88bb8b22f8f2030982764303175dd77b97a42a
-  data.tar.gz: e7b65b4cc79b92b3aa89596fc3e1253fa7897ace
+  metadata.gz: 485925060323de88d131e5d28c6646faa7c1d066
+  data.tar.gz: 44ac8c06b82a3d893bb3e8a9f0f1f33ebd847984
 SHA512:
-  metadata.gz: 7cc74b699dc85ec17ff45ab2d991f77219b7fa93ef03d2d444b21d36b83c47689b4d08ebe81dfc0e428134f9a380262367fd8d3cd33987b831d8628e64a79713
-  data.tar.gz: e2cb3d98ec5b678f67d94c8b70854f2b46c5db7544cfbe1f2713653229195c1c57ee2b28c8fe423ab345a01f16b4b7d68337ebfd4c93719512e68c7c35e7a1e2
+  metadata.gz: 5952f424c3b9f40cdb0929c63860d6c27faf16c611313ad13f159adf30372e94a7fd0e93b7674f401fb6345829cd357a6d1096f0fb56f391c0d07007385812ec
+  data.tar.gz: 8676e815498a52bb0a25b0ccb2918ee08de2a2c8506b1007da03e40bf5a1db5d85a826c21412d645d3455900e5259dbb148b98711d1c4f102457fc41c7527139

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.5.0 (2016-05-31)
+
+Enhancements:
+
+* Support src (rename or copy columns) for JSONPath (but only partially)
+
 # 0.5.0.pre1 (2016-05-24)
 
 Enhancements:

data/README.md

@@ -129,7 +129,8 @@ NOTE:
 
 * JSONPath syntax is not fully supported
 * Embulk's type: json cannot have timestamp column, so `type: timesatmp` for `add_columns` or `columns` with default is not available
-* `src` for `add_columns` or `columns` is not supported yet
+* `src` (to rename or copy columns) for `add_columns` or `columns` is only partially supported yet
+  * the json path directory must be same, for example, `{name: $.foo.copy, src: $foo.bar}` works, but `{name: $foo.copy, src: $.bar.baz}` does not work
 
 ## ToDo
 

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.5.0.pre1"
+version = "0.5.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/add_columns.yml

@@ -23,5 +23,8 @@ filters:
       - {name: d, type: timestamp, default: "2015-07-13"}
       - {name: t, type: timestamp, default: "2015-07-13 00:00:00", timezone: "UTC", format: "%Y-%m-%d %H:%M:%S"}
       - {name: copy_score, src: score}
+      - {name: $.json.foo, type: long, default: 1}
+      - {name: $.json.d, type: string, default: "2015-07-13"}
+      - {name: $.json.copy_foo, src: $.json.foo}
 out:
   type: stdout

data/example/columns.yml

@@ -1,10 +1,3 @@
-# in:
-#   type: random
-#   rows: 100
-#   schema:
-#     id: primary_key
-#     name: string
-#     score: integer
 in:
   type: file
   path_prefix: example/example.csv
@@ -31,5 +24,6 @@ filters:
       - {name: copy_score, src: score}
       - {name: json, default: "{\"foo\":\"FOO\"}"}
       - {name: $.json.foo}
+      - {name: $.json.copy_foo, src: $.json.foo}
 out:
   type: stdout

data/example/drop_columns.yml

@@ -19,5 +19,6 @@ filters:
     drop_columns:
       - {name: time }
       - {name: id }
+      - {name: $.json.foo }
 out:
   type: stdout

data/example/example.yml

@@ -1,10 +1,3 @@
-# in:
-#   type: random
-#   rows: 100
-#   schema:
-#     id: primary_key
-#     name: string
-#     score: integer
 in:
   type: file
   path_prefix: example/example.csv
@@ -31,5 +24,6 @@ filters:
       - {name: copy_score, src: score}
       - {name: json, default: "{\"foo\":\"FOO\"}"}
       - {name: $.json.foo}
+      - {name: $.json.copy_foo, src: $.json.foo}
 out:
   type: stdout

data/src/main/java/org/embulk/filter/column/JsonColumn.java

@@ -1,5 +1,6 @@
 package org.embulk.filter.column;
 
+import org.embulk.config.ConfigException;
 import org.embulk.spi.type.Type;
 import org.msgpack.value.StringValue;
 import org.msgpack.value.Value;
@@ -7,40 +8,60 @@ import org.msgpack.value.ValueFactory;
 
 public class JsonColumn
 {
-    private final String name;
+    private final String path;
     private final Type type;
     private final Value defaultValue;
-    private String objectPath = null; // object path (like directory) of json path
-    private String elementPath = null; // element path (like leaf) of json path
-    private StringValue nameValue = null;
-    private StringValue objectPathValue = null;
-    private StringValue elementPathValue = null;
+    private final String src;
 
-    public JsonColumn(
-            String name,
-            Type type)
+    private StringValue pathValue = null;
+    private String parentPath = null;
+    private String baseName = null;
+    private StringValue parentPathValue = null;
+    private StringValue baseNameValue = null;
+
+    private StringValue srcValue = null;
+    private String srcParentPath = null;
+    private String srcBaseName = null;
+    private StringValue srcParentPathValue = null;
+    private StringValue srcBaseNameValue = null;
+
+    public JsonColumn(String path, Type type)
     {
-        this(name, type, null);
+        this(path, type, null, null);
     }
 
-    public JsonColumn(
-            String name,
-            Type type,
-            Value defaultValue)
+    public JsonColumn(String path, Type type, Value defaultValue)
     {
-        this.name = name;
+        this(path, type, defaultValue, null);
+    }
+
+    public JsonColumn(String path, Type type, Value defaultValue, String src)
+    {
+        this.path = path;
         this.type = type;
         this.defaultValue = (defaultValue == null ? ValueFactory.newNil() : defaultValue);
-        this.objectPath = objectPath(name);
-        this.elementPath = elementPath(name);
-        this.nameValue = ValueFactory.newString(name);
-        this.objectPathValue = ValueFactory.newString(objectPath);
-        this.elementPathValue = ValueFactory.newString(elementPath);
+        this.src = (src == null ? path : src);
+
+        this.pathValue = ValueFactory.newString(path);
+        this.parentPath = parentPath(path);
+        this.baseName = baseName(path);
+        this.parentPathValue = ValueFactory.newString(parentPath);
+        this.baseNameValue = ValueFactory.newString(baseName);
+
+        this.srcValue = ValueFactory.newString(this.src);
+        this.srcParentPath = parentPath(this.src);
+        this.srcBaseName = baseName(this.src);
+        this.srcParentPathValue = ValueFactory.newString(this.srcParentPath);
+        this.srcBaseNameValue = ValueFactory.newString(this.srcBaseName);
+
+        if (! srcParentPath.equals(parentPath)) {
+            throw new ConfigException(String.format("The branch (parent path) of src \"%s\" must be same with of name \"%s\" yet", src, path));
+        }
     }
 
-    public String getName()
+    public String getPath()
     {
-        return name;
+        return path;
     }
 
     public Type getType()
@@ -53,32 +74,63 @@ public class JsonColumn
         return defaultValue;
     }
 
-    public String getObjectPath()
+    public String getSrc()
+    {
+        return src;
+    }
+
+    public StringValue getPathValue()
+    {
+        return pathValue;
+    }
+
+    public String getParentPath()
+    {
+        return parentPath;
+    }
+
+    public String getBaseName()
+    {
+        return baseName;
+    }
+
+    public StringValue getParentPathValue()
+    {
+        return parentPathValue;
+    }
+
+    public StringValue getBaseNameValue()
+    {
+        return baseNameValue;
+    }
+
+    public StringValue getSrcValue()
     {
-        return objectPath;
+        return srcValue;
     }
 
-    public String getElementPath()
+    public String getSrcParentPath()
     {
-        return elementPath;
+        return srcParentPath;
     }
 
-    public StringValue getNameValue()
+    public String getSrcBaseName()
     {
-        return nameValue;
+        return srcBaseName;
     }
 
-    public StringValue getObjectPathValue()
+    public StringValue getSrcParentPathValue()
     {
-        return objectPathValue;
+        return srcParentPathValue;
     }
 
-    public StringValue getElementPathValue()
+    public StringValue getSrcBaseNameValue()
     {
-        return elementPathValue;
+        return srcBaseNameValue;
     }
 
-    public static String objectPath(String path)
+    // like File.dirname
+    public static String parentPath(String path)
     {
         String[] parts = path.split("\\.");
         StringBuilder builder = new StringBuilder();
@@ -96,7 +148,7 @@ public class JsonColumn
         return builder.toString();
     }
 
-    public static String elementPath(String path)
+    public static String baseName(String path)
     {
         String[] parts = path.split("\\.");
         return parts[parts.length - 1];

data/src/main/java/org/embulk/filter/column/JsonVisitor.java

@@ -80,29 +80,29 @@ public class JsonVisitor
 
     private void jsonColumnsPut(String path, JsonColumn value)
     {
-        String objectPath = JsonColumn.objectPath(path);
-        if (! jsonColumns.containsKey(objectPath)) {
-            jsonColumns.put(objectPath, new LinkedHashMap<String, JsonColumn>());
+        String parentPath = JsonColumn.parentPath(path);
+        if (! jsonColumns.containsKey(parentPath)) {
+            jsonColumns.put(parentPath, new LinkedHashMap<String, JsonColumn>());
         }
-        jsonColumns.get(objectPath).put(path, value);
+        jsonColumns.get(parentPath).put(path, value);
     }
 
     private void jsonAddColumnsPut(String path, JsonColumn value)
     {
-        String objectPath = JsonColumn.objectPath(path);
-        if (! jsonAddColumns.containsKey(objectPath)) {
-            jsonAddColumns.put(objectPath, new LinkedHashMap<String, JsonColumn>());
+        String parentPath = JsonColumn.parentPath(path);
+        if (! jsonAddColumns.containsKey(parentPath)) {
+            jsonAddColumns.put(parentPath, new LinkedHashMap<String, JsonColumn>());
         }
-        jsonAddColumns.get(objectPath).put(path, value);
+        jsonAddColumns.get(parentPath).put(path, value);
     }
 
     private void jsonDropColumnsPut(String path)
     {
-        String objectPath = JsonColumn.objectPath(path);
-        if (! jsonDropColumns.containsKey(objectPath)) {
-            jsonDropColumns.put(objectPath, new HashSet<String>());
+        String parentPath = JsonColumn.parentPath(path);
+        if (! jsonDropColumns.containsKey(parentPath)) {
+            jsonDropColumns.put(parentPath, new HashSet<String>());
         }
-        jsonDropColumns.get(objectPath).add(path);
+        jsonDropColumns.get(parentPath).add(path);
     }
 
     // build jsonColumns, jsonAddColumns, and jsonDropColumns
@@ -131,7 +131,8 @@ public class JsonVisitor
                     continue;
                 }
                 if (column.getSrc().isPresent()) {
-                    throw new ConfigException(String.format("columns: src is not supported for json path yet: '%s'", name));
+                    String src = column.getSrc().get();
+                    jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
                 }
                 else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
                     Type type = column.getType().get();
@@ -154,7 +155,8 @@ public class JsonVisitor
                     continue;
                 }
                 if (column.getSrc().isPresent()) {
-                    throw new ConfigException(String.format("add_columns: src is not supported for json path yet: '%s'", name));
+                    String src = column.getSrc().get();
+                    jsonAddColumnsPut(name, new JsonColumn(name, null, null, src));
                 }
                 else if (column.getType().isPresent() && column.getDefault().isPresent()) { // add column
                     Type type = column.getType().get();
@@ -162,7 +164,7 @@ public class JsonVisitor
                     jsonAddColumnsPut(name, new JsonColumn(name, type, defaultValue));
                 }
                 else {
-                    throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"type\" and \"default\"", name));
+                    throw new SchemaConfigException(String.format("add_columns: Column '%s' does not have \"src\", or \"type\" and \"default\"", name));
                 }
             }
         }
@@ -279,15 +281,16 @@ public class JsonVisitor
         }
         else if (this.jsonColumns.containsKey(rootPath)) {
             Map<Value, Value> map = mapValue.map();
-            for (JsonColumn jsonColumn : jsonColumns.get(rootPath).values()) {
-                Value k = jsonColumn.getElementPathValue();
-                Value v = map.get(k);
-                String newPath = jsonColumn.getName();
+            LinkedHashMap<String, JsonColumn> jsonColumns = this.jsonColumns.get(rootPath);
+            for (JsonColumn jsonColumn : jsonColumns.values()) {
+                Value src = jsonColumn.getSrcBaseNameValue();
+                Value v = map.get(src);
+                String newPath = jsonColumn.getPath();
                 Value visited = visit(newPath, v);
                 if (visited == null) {
                     visited = jsonColumn.getDefaultValue();
                 }
-                newValue.add(i++, k);
+                newValue.add(i++, jsonColumn.getPathValue());
                 newValue.add(i++, visited);
             }
         }
@@ -302,9 +305,16 @@ public class JsonVisitor
             }
         }
         if (this.jsonAddColumns.containsKey(rootPath)) {
-            for (JsonColumn jsonColumn : this.jsonAddColumns.get(rootPath).values()) {
-                newValue.add(i++, jsonColumn.getElementPathValue());
-                newValue.add(i++, jsonColumn.getDefaultValue());
+            Map<Value, Value> map = mapValue.map();
+            LinkedHashMap<String, JsonColumn> jsonAddColumns = this.jsonAddColumns.get(rootPath);
+            for (JsonColumn jsonColumn : jsonAddColumns.values()) {
+                Value src = jsonColumn.getSrcBaseNameValue();
+                Value v = map.get(src);
+                if (v == null) {
+                    v = jsonColumn.getDefaultValue();
+                }
+                newValue.add(i++, jsonColumn.getPathValue());
+                newValue.add(i++, v);
             }
         }
         return ValueFactory.newMap(newValue.toArray(new Value[0]), true);

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-column
 version: !ruby/object:Gem::Version
-  version: 0.5.0.pre1
+  version: 0.5.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-05-24 00:00:00.000000000 Z
+date: 2016-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,9 +57,6 @@ files:
 - example/drop_columns.yml
 - example/example.csv
 - example/example.yml
-- example/json_add_columns.yml
-- example/json_columns.yml
-- example/json_drop_columns.yml
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -71,7 +68,7 @@ files:
 - src/main/java/org/embulk/filter/column/JsonColumn.java
 - src/main/java/org/embulk/filter/column/JsonVisitor.java
 - src/test/java/org/embulk/filter/TestColumnFilterPlugin.java
-- classpath/embulk-filter-column-0.5.0.pre1.jar
+- classpath/embulk-filter-column-0.5.0.jar
 homepage: https://github.com/sonots/embulk-filter-column
 licenses:
 - MIT
@@ -87,9 +84,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project:
 rubygems_version: 2.1.9

data/example/json_add_columns.yml

@@ -1,31 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
-      - {name: id, type: long}
-      - {name: name, type: string}
-      - {name: score, type: double}
-      - {name: json, type: json}
-filters:
-  - type: column
-    default_timezone: "Asia/Tokyo"
-    default_timestamp_format: "%Y-%m-%d"
-    columns:
-      - {name: time}
-      - {name: id}
-      - {name: name}
-      - {name: score}
-      - {name: json, default: "{}"}
-    add_columns:
-      - {name: $.json.foo, type: long, default: 1}
-      - {name: $.json.d, type: string, default: "2015-07-13"}
-out:
-  type: stdout

data/example/json_columns.yml

@@ -1,23 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
-      - {name: id, type: long}
-      - {name: name, type: string}
-      - {name: score, type: double}
-      - {name: json, type: json}
-filters:
-  - type: column
-    columns:
-      - {name: json, default: "{\"foo\":\"FOO\"}"}
-      - {name: $.json.foo}
-out:
-  type: stdout

data/example/json_drop_columns.yml

@@ -1,22 +0,0 @@
-in:
-  type: file
-  path_prefix: example/example.csv
-  parser:
-    type: csv
-    charset: UTF-8
-    newline: CRLF
-    null_string: 'NULL'
-    skip_header_lines: 1
-    comment_line_marker: '#'
-    columns:
-      - {name: time,  type: timestamp, format: "%Y-%m-%d"}
-      - {name: id, type: long}
-      - {name: name, type: string}
-      - {name: score, type: double}
-      - {name: json, type: json}
-filters:
-  - type: column
-    drop_columns:
-      - {name: $.json.foo }
-out:
-  type: stdout