embulk-filter-expand_json 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5a19aef396456b82d996d4b61303e1fbc6f69752
-  data.tar.gz: cbb34a591fc08f2e10e5375ae61172be713e9f86
+  metadata.gz: 021e96f2218ac3326f0c470f860bde498866eff9
+  data.tar.gz: dd085b630641bed9a2829b26af1bb03e5ad75680
 SHA512:
-  metadata.gz: b4dd221031f96058833f7a8609b81ba69c645df53eef281dfcbdb10f28055c897c062f839f6daead64bc2a40a8f5775a3101a93683b7779e3d927976db692af3
-  data.tar.gz: d6b96f43d2c49a96d312a4c9dc292f80f9d126fb8331a6293f720252f5c911b2e34c7c0f24c4aad9ea7bd51c38acaa7aab32667cd184fa00cd74ecbe0b9e1b20
+  metadata.gz: 4d6a8fc2b5b53e7b31d8246b231a48ff796cb4d74fcc0cadb713ff22d990327f623e337e56a97c0e0c0ffeb3b1027171199a1193962a6d6854ca552c0de78a2a
+  data.tar.gz: d3ad561c853b5677aa1e968007086dd1bcaf7cf6e116119b0b31906d7dd5922bf8c9920bdbdd74f2c6fcfdaa6dc0c8ffcf88c2545611189b2b734d8fcf416756

data/README.md

@@ -9,27 +9,41 @@ expand columns having json into multiple columns
 ## Configuration
 
 - **json_column_name**: a column name having json to be expanded (string, required)
+- **root**: root property to start fetching each entries, specify in [JsonPath](http://goessner.net/articles/JsonPath/) style (string, default: `\"$.\"`)
 - **expanded_columns**: columns expanded into multiple columns (array of hash, required)
   - **name**: name of the column. you can define [JsonPath](http://goessner.net/articles/JsonPath/) style.
   - **type**: type of the column (see below)
   - **format**: format of the timestamp if type is timestamp
 
+---
+**type of the column**
+
+|name|description|
+|:---|:---|
+|boolean|true or false|
+|long|64-bit signed integers|
+|timestamp|Date and time with nano-seconds precision|
+|double|64-bit floating point numbers|
+|string|Strings|
+
+
 ## Example
 
 ```yaml
 filters:
   - type: expand_json
     json_column_name: json_payload
+    root: "$."
     expanded_columns:
-      - {name: "$.phone_numbers", type: string}
-      - {name: "$.app_id", type: long}
-      - {name: "$.point", type: double}
-      - {name: "$.created_at", type: timestamp, format: "%Y-%m-%d"}
-      - {name: "$.profile.anniversary.et", type: string}
-      - {name: "$.profile.anniversary.voluptatem", type: string}
-      - {name: "$.profile.like_words[1]", type: string}
-      - {name: "$.profile.like_words[2]", type: string}
-      - {name: "$.profile.like_words[0]", type: string}
+      - {name: "phone_numbers", type: string}
+      - {name: "app_id", type: long}
+      - {name: "point", type: double}
+      - {name: "created_at", type: timestamp, format: "%Y-%m-%d"}
+      - {name: "profile.anniversary.et", type: string}
+      - {name: "profile.anniversary.voluptatem", type: string}
+      - {name: "profile.like_words[1]", type: string}
+      - {name: "profile.like_words[2]", type: string}
+      - {name: "profile.like_words[0]", type: string}
 ```
 
 ## Note

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.0.2"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/config.yml

@@ -19,16 +19,17 @@ in:
 filters:
   - type: expand_json
     json_column_name: json_payload
+    root: "$."
     expanded_columns:
-      - {name: "$.phone_numbers", type: string}
-      - {name: "$.app_id", type: long}
-      - {name: "$.point", type: double}
-      - {name: "$.created_at", type: timestamp, format: "%Y-%m-%d"}
-      - {name: "$.profile.anniversary.et", type: string}
-      - {name: "$.profile.anniversary", type: string}
-      - {name: "$.profile.like_words[1]", type: string}
-      - {name: "$.profile.like_words[2]", type: string}
-      - {name: "$.profile.like_words", type: string}
+      - {name: "phone_numbers", type: string}
+      - {name: "app_id", type: long}
+      - {name: "point", type: double}
+      - {name: "created_at", type: timestamp, format: "%Y-%m-%d"}
+      - {name: "profile.anniversary.et", type: string}
+      - {name: "profile.anniversary", type: string}
+      - {name: "profile.like_words[1]", type: string}
+      - {name: "profile.like_words[2]", type: string}
+      - {name: "profile.like_words", type: string}
 
 out:
   type: stdout

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -45,6 +45,10 @@ public class ExpandJsonFilterPlugin
         @Config("json_column_name")
         public String getJsonColumnName();
 
+        @Config("root")
+        @ConfigDefault("\"$.\"")
+        public String getRoot();
+
         @Config("expanded_columns")
         public List<ColumnConfig> getExpandedColumns();
 
@@ -52,10 +56,6 @@ public class ExpandJsonFilterPlugin
         @ConfigDefault("\"UTC\"")
         public String getTimeZone();
 
-//        TODO if needed: add the original column name as the prefix of expanded
-//        @Config("add_original_column_name_as_prefix")
-//        @ConfigDefault("false")
-//        public String getOption2();
     }
 
     @Override
@@ -63,11 +63,7 @@ public class ExpandJsonFilterPlugin
             FilterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-
-        Schema outputSchema = buildOutputSchema(inputSchema,
-                                                task.getJsonColumnName(),
-                                                task.getExpandedColumns());
-
+        Schema outputSchema = buildOutputSchema(task, inputSchema);
         control.run(task.dump(), outputSchema);
     }
 
@@ -76,173 +72,20 @@ public class ExpandJsonFilterPlugin
             final Schema outputSchema, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final List<Column> inputColumns = inputSchema.getColumns();
-
-        final List<Column> inputColumnsExceptExpandedJsonColumn = new ArrayList<>();
-        final List<Column> expandedJsonColumns = new ArrayList<>();
-
-        for (Column column : outputSchema.getColumns()) {
-            if (inputColumns.contains(column)) {
-                inputColumnsExceptExpandedJsonColumn.add(column);
-            }
-            else {
-                expandedJsonColumns.add(column);
-            }
-        }
-
-        Column temporaryJsonColumn = null;
-        for (Column column: inputColumns) {
-            if (column.getName().contentEquals(task.getJsonColumnName())) {
-                temporaryJsonColumn = column;
-            }
-        }
-        final Column jsonColumn = temporaryJsonColumn;
-
-        final HashMap<String, TimestampParser> timestampParserMap = buildTimestampParserMap(task.getJRuby(),
-                                                                                            task.getExpandedColumns(),
-                                                                                            task.getTimeZone());
-        return new PageOutput()
-        {
-            private PageReader pageReader = new PageReader(inputSchema);
-
-            @Override
-            public void add(Page page)
-            {
-                try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output)) {
-                    pageReader.setPage(page);
-
-                    while (pageReader.nextRecord()) {
-                        setInputColumnsExceptFlattenJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
-                        setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserMap);
-                        pageBuilder.addRecord();
-                    }
-                    pageBuilder.finish();
-                }
-                catch (JsonProcessingException e) {
-                    logger.error(e.getMessage());
-                    throw Throwables.propagate(e);
-                }
-            }
-
-            @Override
-            public void finish()
-            {
-                output.finish();
-            }
-
-            @Override
-            public void close()
-            {
-                pageReader.close();
-                output.close();
-            }
-
-            private void setInputColumnsExceptFlattenJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
-                for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
-                    if (pageReader.isNull(inputColumn)) {
-                        pageBuilder.setNull(inputColumn);
-                        continue;
-                    }
-
-                    if (Types.STRING.equals(inputColumn.getType())) {
-                        pageBuilder.setString(inputColumn, pageReader.getString(inputColumn));
-                    }
-                    else if (Types.BOOLEAN.equals(inputColumn.getType())) {
-                        pageBuilder.setBoolean(inputColumn, pageReader.getBoolean(inputColumn));
-                    }
-                    else if (Types.DOUBLE.equals(inputColumn.getType())) {
-                        pageBuilder.setDouble(inputColumn, pageReader.getDouble(inputColumn));
-                    }
-                    else if (Types.LONG.equals(inputColumn.getType())) {
-                        pageBuilder.setLong(inputColumn, pageReader.getLong(inputColumn));
-                    }
-                    else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
-                        pageBuilder.setTimestamp(inputColumn, pageReader.getTimestamp(inputColumn));
-                    }
-                }
-            }
-
-            private void setExpandedJsonColumns(PageBuilder pageBuilder, Column originalJsonColumn, List<Column> expandedJsonColumns, HashMap<String, TimestampParser> timestampParserMap)
-                    throws JsonProcessingException
-            {
-                final ReadContext json;
-                if (pageReader.isNull(originalJsonColumn)) {
-                    json = null;
-                }
-                else {
-                    String jsonObject = pageReader.getString(originalJsonColumn);
-                    Configuration conf = Configuration.defaultConfiguration();
-                    conf = conf.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
-                    json = JsonPath.using(conf).parse(jsonObject);
-                }
-
-                for (Column expandedJsonColumn: expandedJsonColumns) {
-                    if (json == null) {
-                        pageBuilder.setNull(expandedJsonColumn);
-                        continue;
-                    }
-
-                    Object value = json.read(expandedJsonColumn.getName());
-                    final String finalValue = writeJsonPathValueAsString(value);
-                    if (finalValue == null) {
-                        pageBuilder.setNull(expandedJsonColumn);
-                        continue;
-                    }
-
-                    if (Types.STRING.equals(expandedJsonColumn.getType())) {
-                        pageBuilder.setString(expandedJsonColumn, finalValue);
-                    }
-                    else if (Types.BOOLEAN.equals(expandedJsonColumn.getType())) {
-                        pageBuilder.setBoolean(expandedJsonColumn, Boolean.parseBoolean(finalValue));
-                    }
-                    else if (Types.DOUBLE.equals(expandedJsonColumn.getType())) {
-                        pageBuilder.setDouble(expandedJsonColumn, Double.parseDouble(finalValue));
-                    }
-                    else if (Types.LONG.equals(expandedJsonColumn.getType())) {
-                        pageBuilder.setLong(expandedJsonColumn, Long.parseLong(finalValue));
-                    }
-                    else if (Types.TIMESTAMP.equals(expandedJsonColumn.getType())) {
-                        TimestampParser parser = timestampParserMap.get(expandedJsonColumn.getName());
-                        pageBuilder.setTimestamp(expandedJsonColumn, parser.parse(finalValue));
-                    }
-                }
-            }
-
-            private String writeJsonPathValueAsString(Object value)
-                    throws JsonProcessingException
-            {
-                if (value == null) {
-                    return null;
-                }
-                else if (value instanceof List) {
-                    return new ObjectMapper().writeValueAsString(value);
-                }
-                else if (value instanceof Map) {
-                    return new ObjectMapper().writeValueAsString(value);
-                }
-                else if (value instanceof String) {
-                    return (String) value;
-                }
-                else {
-                    return String.valueOf(value);
-                }
-            }
-
-        };
+        return new FilteredPageOutput(task, inputSchema, outputSchema, output);
     }
 
-    private Schema buildOutputSchema(Schema inputSchema, String jsonColumnName, List<ColumnConfig> expandedColumnConfigs)
+    private Schema buildOutputSchema(PluginTask task, Schema inputSchema)
     {
         ImmutableList.Builder<Column> builder = ImmutableList.builder();
 
         int i = 0; // columns index
         for (Column inputColumn: inputSchema.getColumns()) {
-            if (inputColumn.getName().contentEquals(jsonColumnName)) {
+            if (inputColumn.getName().contentEquals(task.getJsonColumnName())) {
                 logger.info("removed column: name: {}, type: {}",
                             inputColumn.getName(),
                             inputColumn.getType());
-                for (ColumnConfig expandedColumnConfig: expandedColumnConfigs) {
+                for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
                     logger.info("added column: name: {}, type: {}, options: {}",
                                 expandedColumnConfig.getName(),
                                 expandedColumnConfig.getType(),
@@ -264,21 +107,4 @@ public class ExpandJsonFilterPlugin
         return new Schema(builder.build());
     }
 
-    private HashMap<String, TimestampParser> buildTimestampParserMap(ScriptingContainer jruby, List<ColumnConfig> expandedColumnConfigs, String timeZone)
-    {
-        final HashMap<String, TimestampParser> timestampParserMap = Maps.newHashMap();
-        for (ColumnConfig expandedColumnConfig: expandedColumnConfigs) {
-            if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
-                String format = expandedColumnConfig.getOption().get(String.class, "format");
-                DateTimeZone timezone = DateTimeZone.forID(timeZone);
-                TimestampParser parser = new TimestampParser(jruby, format, timezone);
-
-                String columnName = expandedColumnConfig.getName();
-
-                timestampParserMap.put(columnName, parser);
-            }
-        }
-
-        return timestampParserMap;
-    }
 }

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -0,0 +1,223 @@
+package org.embulk.filter.expand_json;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.jayway.jsonpath.Configuration;
+import com.jayway.jsonpath.JsonPath;
+import com.jayway.jsonpath.Option;
+import com.jayway.jsonpath.ReadContext;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Types;
+import org.joda.time.DateTimeZone;
+import org.slf4j.Logger;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.*;
+
+/**
+ * Created by takahiro.nakayama on 10/19/15.
+ */
+public class FilteredPageOutput
+    implements PageOutput
+{
+    private final Logger logger = Exec.getLogger(FilteredPageOutput.class);
+    private final String jsonPathRoot;
+    private final List<Column> inputColumnsExceptExpandedJsonColumn;
+    private final List<Column> expandedJsonColumns;
+    private final HashMap<String, TimestampParser> timestampParserHashMap;
+    private final Column jsonColumn;
+    private final PageReader pageReader;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final PageOutput pageOutput;
+
+    FilteredPageOutput(PluginTask task, Schema inputSchema, Schema outputSchema, PageOutput pageOutput)
+    {
+        this.jsonPathRoot = task.getRoot();
+
+        ImmutableList.Builder<Column> inputColumnsExceptExpandedJsonColumnBuilder = ImmutableList.builder();
+        ImmutableList.Builder<Column> expandedJsonColumnsBuilder = ImmutableList.builder();
+        for (Column column : outputSchema.getColumns()) {
+            if (inputSchema.getColumns().contains(column)) {
+                inputColumnsExceptExpandedJsonColumnBuilder.add(column);
+            }
+            else {
+                expandedJsonColumnsBuilder.add(column);
+            }
+        }
+        this.inputColumnsExceptExpandedJsonColumn = inputColumnsExceptExpandedJsonColumnBuilder.build();
+        this.expandedJsonColumns = expandedJsonColumnsBuilder.build();
+
+        Column temporaryJsonColumn = null;
+        for (Column column: inputSchema.getColumns()) {
+            if (column.getName().contentEquals(task.getJsonColumnName())) {
+                temporaryJsonColumn = column;
+            }
+        }
+        this.jsonColumn = temporaryJsonColumn;
+
+        this.timestampParserHashMap = buildTimestampParserHashMap(task);
+        this.pageReader = new PageReader(inputSchema);
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.pageOutput = pageOutput;
+    }
+
+    @Override
+    public void add(Page page)
+    {
+        try (PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, pageOutput)) {
+            pageReader.setPage(page);
+
+            while (pageReader.nextRecord()) {
+                setInputColumnsExceptFlattenJsonColumns(pageBuilder, inputColumnsExceptExpandedJsonColumn);
+                setExpandedJsonColumns(pageBuilder, jsonColumn, expandedJsonColumns, timestampParserHashMap);
+                pageBuilder.addRecord();
+            }
+            pageBuilder.finish();
+        }
+        catch (JsonProcessingException e) {
+            logger.error(e.getMessage());
+            throw Throwables.propagate(e);
+        }
+    }
+
+    @Override
+    public void finish()
+    {
+        pageOutput.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        pageReader.close();
+        pageOutput.close();
+    }
+
+    private HashMap<String, TimestampParser> buildTimestampParserHashMap(PluginTask task)
+    {
+        final HashMap<String, TimestampParser> timestampParserHashMap = Maps.newHashMap();
+        for (ColumnConfig expandedColumnConfig: task.getExpandedColumns()) {
+            if (Types.TIMESTAMP.equals(expandedColumnConfig.getType())) {
+                String format = expandedColumnConfig.getOption().get(String.class, "format");
+                DateTimeZone timezone = DateTimeZone.forID(task.getTimeZone());
+                TimestampParser parser = new TimestampParser(task.getJRuby(), format, timezone);
+
+                String columnName = expandedColumnConfig.getName();
+
+                timestampParserHashMap.put(columnName, parser);
+            }
+        }
+
+        return timestampParserHashMap;
+    }
+    
+    private void setInputColumnsExceptFlattenJsonColumns(PageBuilder pageBuilder, List<Column> inputColumnsExceptExpandedJsonColumn) {
+        for (Column inputColumn: inputColumnsExceptExpandedJsonColumn) {
+            if (pageReader.isNull(inputColumn)) {
+                pageBuilder.setNull(inputColumn);
+                continue;
+            }
+
+            if (Types.STRING.equals(inputColumn.getType())) {
+                pageBuilder.setString(inputColumn, pageReader.getString(inputColumn));
+            }
+            else if (Types.BOOLEAN.equals(inputColumn.getType())) {
+                pageBuilder.setBoolean(inputColumn, pageReader.getBoolean(inputColumn));
+            }
+            else if (Types.DOUBLE.equals(inputColumn.getType())) {
+                pageBuilder.setDouble(inputColumn, pageReader.getDouble(inputColumn));
+            }
+            else if (Types.LONG.equals(inputColumn.getType())) {
+                pageBuilder.setLong(inputColumn, pageReader.getLong(inputColumn));
+            }
+            else if (Types.TIMESTAMP.equals(inputColumn.getType())) {
+                pageBuilder.setTimestamp(inputColumn, pageReader.getTimestamp(inputColumn));
+            }
+        }
+    }
+
+    private void setExpandedJsonColumns(PageBuilder pageBuilder, Column originalJsonColumn, List<Column> expandedJsonColumns, HashMap<String, TimestampParser> timestampParserMap)
+            throws JsonProcessingException
+    {
+        final ReadContext json;
+        if (pageReader.isNull(originalJsonColumn)) {
+            json = null;
+        }
+        else {
+            String jsonObject = pageReader.getString(originalJsonColumn);
+            Configuration conf = Configuration.defaultConfiguration();
+            conf = conf.addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL);
+            conf = conf.addOptions(Option.SUPPRESS_EXCEPTIONS);
+            json = JsonPath.using(conf).parse(jsonObject);
+        }
+
+        for (Column expandedJsonColumn: expandedJsonColumns) {
+            if (json == null) {
+                pageBuilder.setNull(expandedJsonColumn);
+                continue;
+            }
+
+            Object value = json.read(jsonPathRoot + expandedJsonColumn.getName());
+            final String finalValue = writeJsonPathValueAsString(value);
+            if (finalValue == null) {
+                pageBuilder.setNull(expandedJsonColumn);
+                continue;
+            }
+
+            if (Types.STRING.equals(expandedJsonColumn.getType())) {
+                pageBuilder.setString(expandedJsonColumn, finalValue);
+            }
+            else if (Types.BOOLEAN.equals(expandedJsonColumn.getType())) {
+                pageBuilder.setBoolean(expandedJsonColumn, Boolean.parseBoolean(finalValue));
+            }
+            else if (Types.DOUBLE.equals(expandedJsonColumn.getType())) {
+                pageBuilder.setDouble(expandedJsonColumn, Double.parseDouble(finalValue));
+            }
+            else if (Types.LONG.equals(expandedJsonColumn.getType())) {
+                pageBuilder.setLong(expandedJsonColumn, Long.parseLong(finalValue));
+            }
+            else if (Types.TIMESTAMP.equals(expandedJsonColumn.getType())) {
+                TimestampParser parser = timestampParserMap.get(expandedJsonColumn.getName());
+                pageBuilder.setTimestamp(expandedJsonColumn, parser.parse(finalValue));
+            }
+        }
+    }
+
+    private String writeJsonPathValueAsString(Object value)
+            throws JsonProcessingException
+    {
+        if (value == null) {
+            return null;
+        }
+        else if (value instanceof List) {
+            return new ObjectMapper().writeValueAsString(value);
+        }
+        else if (value instanceof Map) {
+            return new ObjectMapper().writeValueAsString(value);
+        }
+        else if (value instanceof String) {
+            return (String) value;
+        }
+        else {
+            return String.valueOf(value);
+        }
+    }
+    
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-expand_json
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-14 00:00:00.000000000 Z
+date: 2015-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,10 +57,11 @@ files:
 - gradlew.bat
 - lib/embulk/filter/expand_json.rb
 - src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java
+- src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java
 - src/test/java/org/embulk/filter/expand_json/TestExpandJsonFilterPlugin.java
 - classpath/asm-1.0.2.jar
 - classpath/asm-3.3.1.jar
-- classpath/embulk-filter-expand_json-0.0.1.jar
+- classpath/embulk-filter-expand_json-0.0.2.jar
 - classpath/json-path-2.0.0.jar
 - classpath/json-smart-2.1.1.jar
 homepage: https://github.com/civitaspo/embulk-filter-expand_json