embulk-filter-expand_json 0.2.2 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c82efb599e6e9e1cd6f02219354383e4d7d1b966
-  data.tar.gz: 1111d92e7fcf2e9df01ac7b0e7e2a27439d71421
+  metadata.gz: 9a6ff1c6a5264a85b4aa62b7a7719c20f5a987c1
+  data.tar.gz: f00862e0f575be8a4f7b1ed938f56370a32d9b0b
 SHA512:
-  metadata.gz: 0e274e928beefad9aacd00b8ef11cbfb5b3772018841d4f7775afd0f03fcb70124ebbea0e5317b401d7dc77b493ab1d56dc445430ebb68a12d7c7798a8fdb2b2
-  data.tar.gz: afebf67519d74a5a86c0e9ad217b2afe66bf7011c6faca6c1f6dc2291ee07afd5f5acb089b168632e2a14b58fc399ceb9ad0dd555923b50e79f9e45acc8e49c3
+  metadata.gz: 787eb4e46c979414d5ee986ec7e1deb2cef85477ac48d0b85aa55762de4dd6ac133d1fa877490e432c393b86cf61d40dac4fd021613bf012ced550200c26d3dc
+  data.tar.gz: 3f3352f3eb7dfd14b506bda85e0c15ac038d5b1495d618eb623ca6f686fc75ec823a28e1d2d1dde6e5e5c8f4f5b0f86169df1cb006a8b796adf8bb90ada6a9b5

data/.travis.yml

@@ -1,8 +1,6 @@
 dist: precise
 language: java
 jdk:
-  - openjdk7
-  - oraclejdk7
   - oraclejdk8
 script:
   - ./gradlew test

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.3.0 (2019-05-02)
+==================
+- [Enhancement] Introduce `cache_provider` option.
+  - https://github.com/civitaspo/embulk-filter-expand_json/pull/41
+
 0.2.2 (2017-09-14)
 ==================
 - [Enhancement] Use TimestampParser's constructor without JRuby ScriptingContainer

data/README.md

@@ -21,6 +21,8 @@ expand columns having json into multiple columns
 - **keep_expanding_json_column**: Not remove the expanding json column from input schema if it's true (false by default)
 - **default_timezone**: Time zone of timestamp columns if values don’t include time zone description (`UTC` by default)
 - **stop_on_invalid_record**: Stop bulk load transaction if an invalid record is included (false by default)
+- **cache_provider**: Cache provider name for JsonPath. `"LRU"` and `"NOOP"` are built-in. You can specify user defined class. (string, default: `"LRU"`)
+  - `"NOOP"` becomes default in the future.
 
 ---
 **type of the column**
@@ -53,6 +55,7 @@ filters:
       - {name: "profile.like_words[0]", type: string}
 ```
 
+
 ## Note
 - If the value evaluated by JsonPath is Array or Hash, the value is written as JSON.
 
@@ -61,13 +64,41 @@ filters:
   - use to evaluate [JsonPath](http://goessner.net/articles/JsonPath/)
   - [Apache License Version 2.0](https://github.com/jayway/JsonPath/blob/master/LICENSE)
 
+## Development
 
-## Build
+### Run Example
+
+```
+./gradlew classpath
+embulk run -Ilib ./example/config.yml
+```
+
+
+### Build
 
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
 ```
 
+## Benchmark for `cache_provider` option
+
+In some cases, `cache_provider: NOOP` improves the performance of this plugin by 3 times (https://github.com/civitaspo/embulk-filter-expand_json/pull/41/).
+So we do a benchmark about `cache_provider`. In our case, `cache_provider: noop` improves the performance by 1.5 times.
+
+|use `expand_json` filter|cache_provider|Time took|records/s|
+|:---|:---|:---|:---|
+|`false`|none|7.62s|1,325,459/s|
+|`true`|`"LRU"`|2m9s|78,025/s|
+|`true`|`"NOOP"`|1m25s|118,476/s|
+
+
+You can reproduce the bench by the below way.
+
+```
+./gradlew classpath
+./bench/run.sh
+```
+
 ## Contributor
 - @Civitaspo
 - @muga

data/bench/.bundle/config

@@ -0,0 +1,2 @@
+---
+BUNDLE_PATH: "vendor/bundle"

data/bench/.gitignore

@@ -0,0 +1,2 @@
+/vendor/
+*.jsonl

data/bench/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'embulk'
+gem 'embulk-parser-none'
+gem 'embulk-filter-speedometer'

data/bench/Gemfile.lock

@@ -0,0 +1,22 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    embulk (0.9.16-java)
+      bundler (>= 1.10.6)
+      liquid (~> 4.0.0)
+      msgpack (~> 1.1.0)
+    embulk-filter-speedometer (0.3.4)
+    embulk-parser-none (0.2.0)
+    liquid (4.0.3)
+    msgpack (1.1.0-java)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  embulk
+  embulk-filter-speedometer
+  embulk-parser-none
+
+BUNDLED WITH
+   1.16.0

data/bench/config_raw.yml

@@ -0,0 +1,14 @@
+in:
+  type: file
+  path_prefix: data.jsonl
+  parser:
+    type: none
+    column_name: payload
+
+filters:
+  - type: speedometer
+    log_interval_seconds: 1
+
+out:
+  type: "null"
+

data/bench/config_with_lru_cache.yml

@@ -0,0 +1,27 @@
+in:
+  type: file
+  path_prefix: data.jsonl
+  parser:
+    type: none
+    column_name: payload
+
+filters:
+  - type: speedometer
+    log_interval_seconds: 1
+  - type: expand_json
+    json_column_name: payload
+    root: "$."
+    expanded_columns:
+      - {name: "phone_numbers", type: string}
+      - {name: "app_id", type: long}
+      - {name: "point", type: double}
+      - {name: "created_at", type: timestamp, format: "%Y-%m-%d"}
+      - {name: "profile.anniversary.et", type: string}
+      - {name: "profile.anniversary", type: string}
+      - {name: "profile.like_words[1]", type: string}
+      - {name: "profile.like_words[2]", type: string}
+      - {name: "profile.like_words", type: string}
+
+out:
+  type: "null"
+

data/bench/config_with_noop_cache.yml

@@ -0,0 +1,28 @@
+in:
+  type: file
+  path_prefix: data.jsonl
+  parser:
+    type: none
+    column_name: payload
+
+filters:
+  - type: speedometer
+    log_interval_seconds: 1
+  - type: expand_json
+    json_column_name: payload
+    cache_provider: noop
+    root: "$."
+    expanded_columns:
+      - {name: "phone_numbers", type: string}
+      - {name: "app_id", type: long}
+      - {name: "point", type: double}
+      - {name: "created_at", type: timestamp, format: "%Y-%m-%d"}
+      - {name: "profile.anniversary.et", type: string}
+      - {name: "profile.anniversary", type: string}
+      - {name: "profile.like_words[1]", type: string}
+      - {name: "profile.like_words[2]", type: string}
+      - {name: "profile.like_words", type: string}
+
+out:
+  type: "null"
+

data/bench/run.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+BENCH_ROOT=$(cd $(dirname $0); pwd)
+DATA_FILE=data.jsonl
+TMP_DATA_FILE=tmp.jsonl
+
+function now() {
+    date +"%FT%T%:z"
+}
+
+echo "[$(now)] Preparing ..."
+(
+    cd $BENCH_ROOT
+    embulk bundle
+
+    if [ -f $DATA_FILE ]; then
+        rm -f $DATA_FILE
+    fi
+    if [ -f $TMP_DATA_FILE ]; then
+        rm -f $TMP_DATA_FILE
+    fi
+    for n in {1..100}; do
+        cat ../example/data.tsv | cut -f5 >> $TMP_DATA_FILE
+    done
+    for n in {1..1000}; do
+        cat $TMP_DATA_FILE >> $DATA_FILE
+    done
+)
+
+echo "[$(now)] Run No expand_json"
+(
+    cd $BENCH_ROOT
+    time embulk run -I ../lib -b . config_raw.yml
+)
+
+echo "[$(now)] Run Default (LRUCache)"
+(
+    cd $BENCH_ROOT
+    time embulk run -I ../lib -b . config_with_lru_cache.yml
+)
+
+echo "[$(now)] Run with NOOPCache"
+(
+    cd $BENCH_ROOT
+    time embulk run -I ../lib -b . config_with_noop_cache.yml
+)
+
+echo "[$(now)] Teardown..."
+(
+    cd $BENCH_ROOT
+    if [ -f $DATA_FILE ]; then
+        rm -f $DATA_FILE
+    fi
+    if [ -f $TMP_DATA_FILE ]; then
+        rm -f $TMP_DATA_FILE
+    fi
+)

data/build.gradle

@@ -1,6 +1,6 @@
 plugins {
     id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.github.jruby-gradle.base" version "1.5.0"
     id "com.github.kt3k.coveralls" version "2.4.0"
     id "jacoco"
     id "java"
@@ -15,16 +15,16 @@ configurations {
     provided
 }
 
-version = "0.2.2"
-sourceCompatibility = 1.7
-targetCompatibility = 1.7
+version = "0.3.0"
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.32"
-    provided "org.embulk:embulk-core:0.8.32"
+    compile  "org.embulk:embulk-core:0.9.+"
+    provided "org.embulk:embulk-core:0.9.+"
     compile "com.jayway.jsonpath:json-path:2.+"
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.32:tests"
+    testCompile "org.embulk:embulk-core:0.9.+:tests"
 }
 
 jacocoTestReport {
@@ -59,19 +59,23 @@ task checkstyle(type: Checkstyle) {
 }
 
 task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "${project.name}.gemspec"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "build", "${project.name}.gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
 task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
-    script "pkg/${project.name}-${project.version}.gem"
+    jrubyArgs "-S"
+    script "gem"
+    scriptArgs "push", "pkg/${project.name}-${project.version}.gem"
 }
 
-task "package"(dependsOn: ["gemspec", "classpath"]) << {
-    println "> Build succeeded."
-    println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+task "package"(dependsOn: ["gemspec", "classpath"]) {
+    doLast {
+        println "> Build succeeded."
+        println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
+    }
 }
 
 task gemspec {
@@ -93,7 +97,6 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")
   spec.require_paths = ["lib"]
 
-  #spec.add_dependency 'YOUR_GEM_DEPENDENCY', ['~> YOUR_GEM_DEPENDENCY_VERSION']
   spec.add_development_dependency 'bundler', ['~> 1.0']
   spec.add_development_dependency 'rake', ['>= 10.0']
 end

data/settings.gradle

@@ -0,0 +1 @@
+rootProject.name = "embulk-filter-expand_json"

data/src/main/java/org/embulk/filter/expand_json/ExpandJsonFilterPlugin.java

@@ -1,6 +1,11 @@
 package org.embulk.filter.expand_json;
 
 import com.google.common.collect.ImmutableList;
+import com.jayway.jsonpath.JsonPathException;
+import com.jayway.jsonpath.spi.cache.Cache;
+import com.jayway.jsonpath.spi.cache.CacheProvider;
+import com.jayway.jsonpath.spi.cache.LRUCache;
+import com.jayway.jsonpath.spi.cache.NOOPCache;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
@@ -19,6 +24,8 @@ import org.slf4j.Logger;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
 
 public class ExpandJsonFilterPlugin
         implements FilterPlugin
@@ -29,14 +36,14 @@ public class ExpandJsonFilterPlugin
             extends Task, TimestampParser.Task
     {
         @Config("json_column_name")
-        public String getJsonColumnName();
+        String getJsonColumnName();
 
         @Config("root")
         @ConfigDefault("\"$.\"")
-        public String getRoot();
+        String getRoot();
 
         @Config("expanded_columns")
-        public List<ColumnConfig> getExpandedColumns();
+        List<ColumnConfig> getExpandedColumns();
 
         // default_timezone option from TimestampParser.Task
 
@@ -46,7 +53,11 @@ public class ExpandJsonFilterPlugin
 
         @Config("keep_expanding_json_column")
         @ConfigDefault("false")
-        public boolean getKeepExpandingJsonColumn();
+        boolean getKeepExpandingJsonColumn();
+
+        @Config("cache_provider")
+        @ConfigDefault("null")
+        Optional<String> getCacheProviderName();
     }
 
     @Override
@@ -60,6 +71,9 @@ public class ExpandJsonFilterPlugin
 
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        // set cache provider
+        task.getCacheProviderName().ifPresent(this::setCacheProvider);
+
         // check if a column specified as json_column_name option exists or not
         Column jsonColumn = inputSchema.lookupColumn(task.getJsonColumnName());
         if (jsonColumn.getType() != Types.STRING && jsonColumn.getType() != Types.JSON) {
@@ -79,6 +93,8 @@ public class ExpandJsonFilterPlugin
             final Schema outputSchema, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
+        // set cache provider for mapreduce executor.
+        task.getCacheProviderName().ifPresent(this::setCacheProviderOrIgnore);
         return new FilteredPageOutput(task, inputSchema, outputSchema, output);
     }
 
@@ -152,4 +168,39 @@ public class ExpandJsonFilterPlugin
             columnList.add(columnName);
         }
     }
+
+    private void setCacheProvider(String cacheProviderName)
+    {
+        String upperCacheProviderName = cacheProviderName.toUpperCase(Locale.ENGLISH);
+        switch (upperCacheProviderName)
+        {
+            case "LRU":
+                CacheProvider.setCache(new LRUCache(400));
+                break;
+
+            case "NOOP":
+                CacheProvider.setCache(new NOOPCache());
+                break;
+
+            default:
+                try {
+                    Class<?> klass = Class.forName(cacheProviderName);
+                    Cache cache = (Cache) klass.newInstance();
+                    CacheProvider.setCache(cache);
+                }
+                catch (ClassNotFoundException | IllegalAccessException | InstantiationException | ClassCastException e) {
+                    throw new ConfigException(String.format("Cache Provider '%s' is not supported: %s.", cacheProviderName, e.getMessage()), e);
+                }
+        }
+    }
+
+    private void setCacheProviderOrIgnore(String cacheProviderName)
+    {
+        try {
+            setCacheProvider(cacheProviderName);
+        }
+        catch (JsonPathException e) {
+            logger.debug("Cache:{} is already set.", CacheProvider.getCache().getClass());
+        }
+    }
 }

data/src/main/java/org/embulk/filter/expand_json/FilteredPageOutput.java

@@ -32,9 +32,6 @@ import java.util.Map;
 
 import static org.embulk.filter.expand_json.ExpandJsonFilterPlugin.PluginTask;
 
-/**
- * Created by takahiro.nakayama on 10/19/15.
- */
 public class FilteredPageOutput
     implements PageOutput
 {