embulk-filter-typecast 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9fa85a581cc7a32dbae99b2fbacac7150ffe930f
-  data.tar.gz: fd78481a512883f60c0c360cb1ffc08335e70fa5
+  metadata.gz: 58d54323014090807f457eec3b84aef29060ddd2
+  data.tar.gz: eb7be6477a3a993ce83da5a3e404fa0e7fd3a50d
 SHA512:
-  metadata.gz: 2dcc900cd25a80f8cb6226e3669a7a35b04065fc1a925fd0bc55ee03ee19c90315e1bacf2e00fe35713e11dc60a4ce50feba962ad4e919a5322c67f8726c6f5b
-  data.tar.gz: 0b800b1434d4099d29fbe7429a97fef8a6dc108dddea0672238528edb06b75f656b07e0a5cc95a377ebab7dad0555a55dcd3068504c2a6dffddb38523614885f
+  metadata.gz: f19f899b820a7faccca9d261b8cc807ac230321b26b4ccde552e7308c6853bc01a24b7cefde15eef9061f1b06e55f0b1b31e8ff67f29ab83d4f07ac3feb488db
+  data.tar.gz: 479bce7de22d8ecefdd6ddde53cd602f44fefdab0394cb0b83df9eb1f67cf2aeb980aa87b24da62f72a3ce47192f13af5ba6b9c1006116fb1da0f64156313ad2

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+# 0.1.5 (2016-11-06)
+
+Enhancements:
+
+* Support jsonpath bracket notation
+
 # 0.1.4 (2016-10-26)
 
 Enhancements:

data/README.md

@@ -19,17 +19,23 @@ A filter plugin for Embulk to cast column type.
 
 See [example.csv](./example/example.csv) and [example.yml](./example/example.yml).
 
-## JSONPath (like) name
+## JSONPath
 
 For `type: json` column, you can specify [JSONPath](http://goessner.net/articles/JsonPath/) for column's name as:
 
 ```
-$.payload.key1
-$.payload.array[0]
-$.payload.array[*]
+name: $.payload.key1
+name: "$.payload.array[0]"
+name: "$.payload.array[*]"
+name: $['payload']['key1.key2']
 ```
 
-NOTE: JSONPath syntax is not fully supported
+Following operators of JSONPath are not supported:
+
+* Multiple properties such as `['name','name']`
+* Multiple array indexes such as `[1,2]`
+* Array slice such as `[1:2]`
+* Filter expression such as `[?(<expression>)]`
 
 ## ToDo
 

data/build.gradle

@@ -13,13 +13,14 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.+"
     provided "org.embulk:embulk-core:0.8.+"
+    compile  "io.github.medjed:JsonPathCompiler:0.1.1"
 
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.+:tests"

data/example/example.yml

@@ -0,0 +1,41 @@
+in:
+  type: file
+  path_prefix: example/example.csv
+  parser:
+    type: csv
+    charset: UTF-8
+    newline: CRLF
+    null_string: ''
+    skip_header_lines: 1
+    comment_line_marker: '#'
+    columns:
+      - {name: timestamp, type: string}
+      - {name: "null",    type: string}
+      - {name: long,      type: string}
+      - {name: string,    type: string}
+      - {name: double,    type: string}
+      - {name: json1,     type: string}
+      - {name: json2,     type: string}
+      - {name: array_str, type: string}
+      - {name: array_int, type: string}
+      - {name: ignore,    type: string}
+      - {name: boolean,   type: string}
+filters:
+  - type: typecast
+    columns:
+      - {name: timestamp, type: timestamp, format: "%Y-%m-%d %H:%M:%S.%N", timezone: "+09:00"}
+      - {name: "null",    type: long}
+      - {name: long,      type: long}
+      - {name: string,    type: string}
+      - {name: double,    type: double}
+      - {name: json1,     type: json}
+      - {name: json2,     type: json}
+      - {name: array_str, type: json}
+      - {name: array_int, type: json}
+      - {name: boolean,   type: boolean}
+      - {name: "$.json1.string", type: long}
+      - {name: "$.json2.long",   type: long}
+      - {name: "$.array_str[0]", type: long}
+      - {name: "$.array_int[*]", type: long}
+out:
+  type: "null"

data/src/main/java/org/embulk/filter/typecast/ColumnCaster.java

@@ -1,5 +1,9 @@
 package org.embulk.filter.typecast;
 
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.Utils;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
 import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
 import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
 
@@ -61,7 +65,7 @@ class ColumnCaster
     {
         // columnName => TimestampParser
         for (ColumnConfig columnConfig : task.getColumns()) {
-            if (columnConfig.getName().startsWith("$.")) {
+            if (PathCompiler.isProbablyJsonPath(columnConfig.getName())) {
                 continue; // type: json columns do not support type: timestamp
             }
             Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
@@ -76,7 +80,7 @@ class ColumnCaster
     {
         // columnName => TimestampFormatter
         for (ColumnConfig columnConfig : task.getColumns()) {
-            if (columnConfig.getName().startsWith("$.")) {
+            if (PathCompiler.isProbablyJsonPath(columnConfig.getName())) {
                 continue; // type: json columns do not have type: timestamp
             }
             Column inputColumn = inputSchema.lookupColumn(columnConfig.getName());
@@ -200,7 +204,8 @@ class ColumnCaster
         }
         else if (outputType instanceof JsonType) {
             Value jsonValue = StringCast.asJson(value);
-            String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
+            String name = outputColumn.getName();
+            String jsonPath = new StringBuilder("$").append(PropertyPathToken.getPathFragment(name)).toString();
             Value castedValue = jsonVisitor.visit(jsonPath, jsonValue);
             pageBuilder.setJson(outputColumn, castedValue);
         }
@@ -238,7 +243,8 @@ class ColumnCaster
 
     public void setFromJson(Column outputColumn, Value value)
     {
-        String jsonPath = new StringBuilder("$.").append(outputColumn.getName()).toString();
+        String name = outputColumn.getName();
+        String jsonPath = new StringBuilder("$").append(PropertyPathToken.getPathFragment(name)).toString();
         Value castedValue = jsonVisitor.visit(jsonPath, value);
         Type outputType = outputColumn.getType();
         if (outputType instanceof BooleanType) {

data/src/main/java/org/embulk/filter/typecast/JsonPathUtil.java

@@ -0,0 +1,78 @@
+package org.embulk.filter.typecast;
+
+import io.github.medjed.jsonpathcompiler.InvalidPathException;
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayIndexOperation;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.FunctionPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PredicatePathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.ScanPathToken;
+import org.embulk.config.ConfigException;
+
+public class JsonPathUtil
+{
+    private JsonPathUtil() {}
+
+    public static String getColumnName(String jsonPath)
+    {
+        Path compiledPath;
+        try {
+            compiledPath = PathCompiler.compile(jsonPath);
+        }
+        catch (InvalidPathException e) {
+            throw new ConfigException(String.format("jsonpath %s, %s", jsonPath, e.getMessage()));
+        }
+        PathToken pathToken = compiledPath.getRoot();
+        pathToken = pathToken.next(); // skip $
+        return ((PropertyPathToken) pathToken).getProperties().get(0);
+    }
+
+    public static void assertJsonPathFormat(String path)
+    {
+        Path compiledPath;
+        try {
+            compiledPath = PathCompiler.compile(path);
+        }
+        catch (InvalidPathException e) {
+            throw new ConfigException(String.format("jsonpath %s, %s", path, e.getMessage()));
+        }
+        PathToken pathToken = compiledPath.getRoot();
+        while (true) {
+            assertSupportedPathToken(pathToken, path);
+            if (pathToken.isLeaf()) {
+                break;
+            }
+            pathToken = pathToken.next();
+        }
+    }
+
+    protected static void assertSupportedPathToken(PathToken pathToken, String path)
+    {
+        if (pathToken instanceof ArrayPathToken) {
+            ArrayIndexOperation arrayIndexOperation = ((ArrayPathToken) pathToken).getArrayIndexOperation();
+            assertSupportedArrayPathToken(arrayIndexOperation, path);
+        }
+        else if (pathToken instanceof ScanPathToken) {
+            throw new ConfigException(String.format("scan path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof FunctionPathToken) {
+            throw new ConfigException(String.format("function path token is not supported \"%s\"", path));
+        }
+        else if (pathToken instanceof PredicatePathToken) {
+            throw new ConfigException(String.format("predicate path token is not supported \"%s\"", path));
+        }
+    }
+
+    protected static void assertSupportedArrayPathToken(ArrayIndexOperation arrayIndexOperation, String path)
+    {
+        if (arrayIndexOperation == null) {
+            throw new ConfigException(String.format("Array Slice Operation is not supported \"%s\"", path));
+        }
+        else if (!arrayIndexOperation.isSingleIndexOperation()) {
+            throw new ConfigException(String.format("Multi Array Indexes is not supported \"%s\"", path));
+        }
+    }
+}

data/src/main/java/org/embulk/filter/typecast/JsonVisitor.java

@@ -1,5 +1,10 @@
 package org.embulk.filter.typecast;
 
+import io.github.medjed.jsonpathcompiler.expressions.Path;
+import io.github.medjed.jsonpathcompiler.expressions.path.ArrayPathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathToken;
+import io.github.medjed.jsonpathcompiler.expressions.path.PropertyPathToken;
 import org.embulk.filter.typecast.TypecastFilterPlugin.ColumnConfig;
 import org.embulk.filter.typecast.TypecastFilterPlugin.PluginTask;
 
@@ -33,20 +38,33 @@ public class JsonVisitor
         this.inputSchema  = inputSchema;
         this.outputSchema = outputSchema;
 
+        assertJsonPathFromat();
         buildShouldVisitSet();
         buildJsonPathTypeMap();
     }
 
+    private void assertJsonPathFromat()
+    {
+        for (ColumnConfig columnConfig : task.getColumns()) {
+            String name = columnConfig.getName();
+            if (! PathCompiler.isProbablyJsonPath(name)) {
+                continue;
+            }
+            JsonPathUtil.assertJsonPathFormat(name);
+        }
+    }
+
     private void buildJsonPathTypeMap()
     {
         // json path => Type
         for (ColumnConfig columnConfig : task.getColumns()) {
             String name = columnConfig.getName();
-            if (!name.startsWith("$.")) {
+            if (! PathCompiler.isProbablyJsonPath(name)) {
                 continue;
             }
+            Path compiledPath = PathCompiler.compile(name);
             Type type = columnConfig.getType();
-            this.jsonPathTypeMap.put(name, type);
+            this.jsonPathTypeMap.put(compiledPath.toString(), type);
         }
     }
 
@@ -55,26 +73,15 @@ public class JsonVisitor
         // json partial path => Boolean to avoid unnecessary type: json visit
         for (ColumnConfig columnConfig : task.getColumns()) {
             String name = columnConfig.getName();
-            if (!name.startsWith("$.")) {
+            if (! PathCompiler.isProbablyJsonPath(name)) {
                 continue;
             }
-            String[] parts = name.split("\\.");
+            PathToken parts = PathCompiler.compile(name).getRoot();
             StringBuilder partialPath = new StringBuilder("$");
-            for (int i = 1; i < parts.length; i++) {
-                if (parts[i].contains("[")) {
-                    String[] arrayParts = parts[i].split("\\[");
-                    partialPath.append(".").append(arrayParts[0]);
-                    this.shouldVisitSet.add(partialPath.toString());
-                    for (int j = 1; j < arrayParts.length; j++) {
-                        // Supports both [0] and [*]
-                        partialPath.append("[").append(arrayParts[j]);
-                        this.shouldVisitSet.add(partialPath.toString());
-                    }
-                }
-                else {
-                    partialPath.append(".").append(parts[i]);
-                    this.shouldVisitSet.add(partialPath.toString());
-                }
+            while (! parts.isLeaf()) {
+                parts = parts.next(); // first next() skips "$"
+                partialPath.append(parts.getPathFragment());
+                this.shouldVisitSet.add(partialPath.toString());
             }
         }
     }
@@ -118,7 +125,8 @@ public class JsonVisitor
             int size = arrayValue.size();
             Value[] newValue = new Value[size];
             for (int i = 0; i < size; i++) {
-                String k = new StringBuilder(rootPath).append("[").append(Integer.toString(i)).append("]").toString();
+                String pathFragment = ArrayPathToken.getPathFragment(i);
+                String k = new StringBuilder(rootPath).append(pathFragment).toString();
                 if (!shouldVisit(k)) {
                     k = new StringBuilder(rootPath).append("[*]").toString(); // try [*] too
                 }
@@ -135,7 +143,8 @@ public class JsonVisitor
             for (Map.Entry<Value, Value> entry : mapValue.entrySet()) {
                 Value k = entry.getKey();
                 Value v = entry.getValue();
-                String newPath = new StringBuilder(rootPath).append(".").append(k.asStringValue().asString()).toString();
+                String pathFragment = PropertyPathToken.getPathFragment(k.asStringValue().asString());
+                String newPath = new StringBuilder(rootPath).append(pathFragment).toString();
                 Value r = visit(newPath, v);
                 newValue[i++] = k;
                 newValue[i++] = r;

data/src/main/java/org/embulk/filter/typecast/TypecastFilterPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.filter.typecast;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import io.github.medjed.jsonpathcompiler.expressions.path.PathCompiler;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
@@ -91,10 +92,10 @@ public class TypecastFilterPlugin implements FilterPlugin
         // throw if column does not exist
         for (ColumnConfig columnConfig : columnConfigs) {
             String name = columnConfig.getName();
-            if (name.startsWith("$.")) { // check only top level column name
-                String firstName = name.split("\\.", 3)[1];
-                String firstNameWithoutArray = firstName.split("\\[")[0];
-                inputSchema.lookupColumn(firstNameWithoutArray);
+            if (PathCompiler.isProbablyJsonPath(name)) {
+                // check only top level column name
+                String columnName = JsonPathUtil.getColumnName(name);
+                inputSchema.lookupColumn(columnName);
             }
             else {
                 inputSchema.lookupColumn(name);
@@ -103,7 +104,7 @@ public class TypecastFilterPlugin implements FilterPlugin
         // throw if timestamp is specified in json path
         for (ColumnConfig columnConfig : columnConfigs) {
             String name = columnConfig.getName();
-            if (name.startsWith("$.") && columnConfig.getType() instanceof TimestampType) {
+            if (PathCompiler.isProbablyJsonPath(name) && columnConfig.getType() instanceof TimestampType) {
                 throw new ConfigException(String.format("embulk-filter-typecast: timestamp type is not supported in json column: \"%s\"", name));
             }
         }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-typecast
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-26 00:00:00.000000000 Z
+date: 2016-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -54,6 +54,7 @@ files:
 - config/checkstyle/checkstyle.xml
 - example/empty.yml
 - example/example.csv
+- example/example.yml
 - example/from_string.txt
 - example/from_string.yml
 - example/jsoncast.json
@@ -70,6 +71,7 @@ files:
 - src/main/java/org/embulk/filter/typecast/ColumnCaster.java
 - src/main/java/org/embulk/filter/typecast/ColumnVisitorImpl.java
 - src/main/java/org/embulk/filter/typecast/JsonCaster.java
+- src/main/java/org/embulk/filter/typecast/JsonPathUtil.java
 - src/main/java/org/embulk/filter/typecast/JsonVisitor.java
 - src/main/java/org/embulk/filter/typecast/TypecastFilterPlugin.java
 - src/main/java/org/embulk/filter/typecast/cast/BooleanCast.java
@@ -84,7 +86,13 @@ files:
 - src/test/java/org/embulk/filter/typecast/cast/TestLongCast.java
 - src/test/java/org/embulk/filter/typecast/cast/TestStringCast.java
 - src/test/java/org/embulk/filter/typecast/cast/TestTimestampCast.java
-- classpath/embulk-filter-typecast-0.1.4.jar
+- classpath/accessors-smart-1.1.jar
+- classpath/asm-5.0.3.jar
+- classpath/commons-lang3-3.4.jar
+- classpath/embulk-filter-typecast-0.1.5.jar
+- classpath/json-smart-2.2.1.jar
+- classpath/JsonPathCompiler-0.1.1.jar
+- classpath/slf4j-api-1.7.21.jar
 homepage: https://github.com/sonots/embulk-filter-typecast
 licenses:
 - MIT