embulk-filter-copy 0.0.2 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d2500f6ad78f1db21f25444a8454312d190d3229
-  data.tar.gz: 6b3f59c7329235aeaff13ee152a235617b282ab1
+  metadata.gz: 8c71648337b5b1edcc9bee9e7839dcaca696cfe2
+  data.tar.gz: 59fd5e9cf1092a9880552074a0e9e404d4fba62c
 SHA512:
-  metadata.gz: 6498b12ec28031b054e31cd14a35a37540942a23fcd6b0dd95149609afa0797a201badadc0ccddecb1baf49dfdfbfbd281dc5da9fa95d875bee69e25be0c90b8
-  data.tar.gz: 0f1552016b5e7417a69a868a73820c26451c7f7e2b13fb0dc773d031be8c70e4fbb660de74fcf8bf32fed9bf28c8adb8f2567379da6a47615284ad94f8db1a22
+  metadata.gz: 0b35ef1db23ce24ffa0c76d82455f3053e57db6c629490880c0e7ec4c33fcb3b906412030cc4974f938522c474aec0a8e29f8b1d085326a455f2da032ec7fc34
+  data.tar.gz: f4dac991cee674b2a71e92539f3d90ca4719a6570e4e10c2fc30d1cf19aadef0a098c2841e9f457cdc23882cf8a1837c1191ac736480ff1153d5d6f86368c74b

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.1.0 (2019-09-26)
+==================
+* Follow embulk v0.9.x interfaces.
+* Packing as MessagePack become faster because, when parsing or formatting `Timestamp` values, use only `epochSecond` and `nanoAdjustment` instead of `TimestampFormatter` and `TimestampParser`.
+
 0.0.2 (2017-07-01)
 ==================
 * Write a default values test.
@@ -5,4 +10,4 @@
 
 0.0.1 (2017-06-07)
 ==================
-* first release.
+* first release.

data/README.md

@@ -2,6 +2,8 @@
 
 Copy records and run another embulk by using them as input data source.
 
+The Document for Japanese is [here](http://qiita.com/Civitaspo/items/da8483c28817071d90dc).
+
 ## Overview
 
 * **Plugin type**: filter
@@ -53,7 +55,9 @@ $ embulk run example/config.yml -Ilib
 
 ### Run test:
 
-TBD
+```shell
+$ ./gradlew test
+```
 
 ### Release gem:
 Fix [build.gradle](./build.gradle), then

data/build.gradle

@@ -14,22 +14,23 @@ configurations {
     provided
 }
 
-version = "0.0.2"
+version = "0.1.0"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.23"
-    provided "org.embulk:embulk-core:0.8.23"
-    compile  "com.okumin:influent-java:0.2.+"
+    compile  "org.embulk:embulk-core:0.9.18"
+    provided "org.embulk:embulk-core:0.9.18"
+    compile 'com.google.guava:guava:23.0'
     // The latest version of fluency is 1.3.0, but
     // This plugin use 1.1.0 for using the same msgpack version as embulk.
     // ref. https://github.com/komamitsu/fluency/compare/1.1.0...1.3.0
     compile  "org.komamitsu:fluency:1.1.0"
+    compile 'com.okumin:influent-java:0.3.0'
 
     testCompile "junit:junit:4.+"
-    testCompile "org.embulk:embulk-core:0.8.23:tests"
+    testCompile "org.embulk:embulk-core:0.9.18:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/example/config.yml

@@ -10,6 +10,7 @@ in:
       - { name: id, type: long }
       - { name: description, type: string }
       - { name: name, type: string }
+      - { name: t, type: timestamp, format: "%Y-%m-%d %H:%M:%S %z"}
       - { name: payload, type: json}
     stop_on_invalid_record: true
 

data/example/data.tsv

@@ -1,5 +1,5 @@
-0	c20ef94602	c212c89f91	{"a":0,"b":"99"}
-1	330a9fc33a	e25b33b616	{"a":1,"b":"a9"}
-2	707b3b7588	90823c6a1f	{"a":2,"b":"96"}
-3	8d8288e66f		{"a":3,"b":"86"}
-4	c54d8b6481	e56a40571c	{"a":4,"b":"d2"}
+0	c20ef94602	c212c89f91	2017-10-24 03:54:35 +0900	{"a":0,"b":"99"}
+1	330a9fc33a	e25b33b616	2017-10-22 19:53:31 +0900	{"a":1,"b":"a9"}
+2	707b3b7588	90823c6a1f	2017-10-23 23:42:43 +0900	{"a":2,"b":"96"}
+3	8d8288e66f		2017-10-22 06:12:13 +0900	{"a":3,"b":"86"}
+4	c54d8b6481	e56a40571c	2017-10-23 04:59:16 +0900	{"a":4,"b":"d2"}

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,5 @@
-#Sun Jan 08 00:35:58 PST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-4.1-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -33,11 +33,11 @@ DEFAULT_JVM_OPTS=""
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
-warn ( ) {
+warn () {
     echo "$*"
 }
 
-die ( ) {
+die () {
     echo
     echo "$*"
     echo
@@ -154,16 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
 
 # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
-if [[ "$(uname)" == "Darwin" ]] && [[ "$HOME" == "$PWD" ]]; then
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
   cd "$(dirname "$0")"
 fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java

@@ -5,21 +5,21 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
-import org.embulk.filter.copy.executor.EmbulkExecutor;
 import org.embulk.filter.copy.forward.InForwardService;
 import org.embulk.filter.copy.forward.OutForwardEventBuilder;
 import org.embulk.filter.copy.forward.OutForwardService;
-import org.embulk.filter.copy.forward.OutForwardVisitor;
 import org.embulk.filter.copy.plugin.InternalForwardInputPlugin;
-import org.embulk.filter.copy.util.StandardColumnVisitor;
+import org.embulk.filter.copy.runner.AsyncEmbulkRunnerService;
+import org.embulk.filter.copy.runner.EmbulkRunner;
+import org.embulk.filter.copy.spi.PageBuilder;
+import org.embulk.filter.copy.spi.PageReader;
+import org.embulk.filter.copy.spi.StandardColumnVisitor;
+import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Page;
-import org.embulk.spi.PageBuilder;
 import org.embulk.spi.PageOutput;
-import org.embulk.spi.PageReader;
 import org.embulk.spi.Schema;
-import org.embulk.spi.time.TimestampFormatter;
 import org.slf4j.Logger;
 
 import java.util.List;
@@ -45,14 +45,13 @@ public class CopyFilterPlugin
     }
 
     public interface PluginTask
-            extends Task, EmbulkExecutor.Task, TimestampFormatter.Task,
-            OutForwardService.Task, InForwardService.Task
+            extends Task, OutForwardService.Task, InForwardService.Task
     {
         @Config("config")
         EmbulkConfig getConfig();
     }
 
-    private ConfigSource configure(PluginTask task, Schema schema)
+    private EmbulkRunner configure(PluginTask task, Schema schema)
     {
         ConfigSource inputConfig = Exec.newConfigSource();
         inputConfig.set("type", InternalForwardInputPlugin.PLUGIN_NAME);
@@ -60,27 +59,21 @@ public class CopyFilterPlugin
         inputConfig.set("message_tag", task.getMessageTag());
         inputConfig.set("shutdown_tag", task.getShutdownTag());
         inputConfig.set("in_forward", task.getInForwardTask());
-        inputConfig.set("default_timestamp_format", task.getDefaultTimestampFormat());
-        inputConfig.set("default_timezone", task.getDefaultTimeZone());
 
-        ConfigSource embulkRunConfig = Exec.newConfigSource();
-        embulkRunConfig.set("exec", task.getConfig().getExecConfig());
-        embulkRunConfig.set("in", inputConfig);
-        embulkRunConfig.set("filters", task.getConfig().getFilterConfig());
-        embulkRunConfig.set("out", task.getConfig().getOutputConfig());
-
-        return embulkRunConfig;
+        return EmbulkRunner.builder()
+                .execConfig(task.getConfig().getExecConfig())
+                .inputConfig(inputConfig)
+                .filterConfig(task.getConfig().getFilterConfig())
+                .outputConfig(task.getConfig().getOutputConfig())
+                .build();
     }
 
-    private void withEmbulkRun(EmbulkExecutor executor, ConfigSource config, Runnable r)
+    private void withEmbulkRun(AsyncEmbulkRunnerService service, Runnable r)
     {
-        executor.setup();
-        executor.executeAsync(config);
-
+        service.startAsync();
+        service.awaitRunning();
         r.run();
-
-        executor.waitUntilExecutionFinished();
-        executor.shutdown();
+        service.awaitTerminated();
     }
 
     @Override
@@ -88,12 +81,12 @@ public class CopyFilterPlugin
             FilterPlugin.Control control)
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        ConfigSource embulkRunConfig = configure(task, inputSchema);
-        EmbulkExecutor embulkExecutor = EmbulkExecutor.buildExecutor(task);
+        EmbulkRunner embulkRunner = configure(task, inputSchema);
+        AsyncEmbulkRunnerService embulkRunnerService = new AsyncEmbulkRunnerService(embulkRunner);
 
-        withEmbulkRun(embulkExecutor, embulkRunConfig, () -> {
-            Schema outputSchema1 = inputSchema;
-            control.run(task.dump(), outputSchema1);
+        withEmbulkRun(embulkRunnerService, () -> {
+            Schema outputSchema = inputSchema;
+            control.run(task.dump(), outputSchema);
             OutForwardService.sendShutdownMessage(task);
         });
     }
@@ -103,10 +96,6 @@ public class CopyFilterPlugin
             final Schema outputSchema, final PageOutput output)
     {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
-        TimestampFormatter timestampFormatter = new TimestampFormatter(
-                task.getJRuby(),
-                task.getDefaultTimestampFormat(),
-                task.getDefaultTimeZone());
 
         return new PageOutput()
         {
@@ -114,8 +103,8 @@ public class CopyFilterPlugin
             private final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), outputSchema, output);
             private final StandardColumnVisitor standardColumnVisitor = new StandardColumnVisitor(pageReader, pageBuilder);
             private final OutForwardService outForwardService = new OutForwardService(task);
-            private final OutForwardEventBuilder eventBuilder = new OutForwardEventBuilder(outputSchema, timestampFormatter);
-            private final OutForwardVisitor outForwardVisitor = new OutForwardVisitor(pageReader, eventBuilder);
+            private final OutForwardEventBuilder eventBuilder = new OutForwardEventBuilder(outputSchema, outForwardService);
+            private final ColumnVisitor outForwardVisitor = new StandardColumnVisitor(pageReader, eventBuilder);
 
             @Override
             public void add(Page page)
@@ -123,7 +112,7 @@ public class CopyFilterPlugin
                 pageReader.setPage(page);
                 while (pageReader.nextRecord()) {
                     outputSchema.visitColumns(outForwardVisitor);
-                    eventBuilder.emitMessage(outForwardService);
+                    eventBuilder.addRecord();
 
                     outputSchema.visitColumns(standardColumnVisitor);
                     pageBuilder.addRecord();

data/src/main/java/org/embulk/filter/copy/forward/{ForwardBaseTask.java → ForwardParentTask.java}

@@ -4,7 +4,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 
-public interface ForwardBaseTask
+public interface ForwardParentTask
     extends Task
 {
     @Config("shutdown_tag")

data/src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java

@@ -1,147 +1,152 @@
 package org.embulk.filter.copy.forward;
 
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Maps;
+import com.google.common.collect.Lists;
 import influent.EventStream;
 import org.embulk.spi.Column;
 import org.embulk.spi.Schema;
-import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampParser;
 import org.msgpack.value.Value;
+import org.embulk.filter.copy.spi.DataReader;
 
-import java.util.Map;
+import java.util.List;
+import java.util.Optional;
 
 public class InForwardEventReader
+        implements DataReader
 {
-    private final Schema schema;
-    private final Map<String, Column> columnMap;
-
-    private final JsonParser jsonParser = new JsonParser();
-    private final TimestampParser timestampParser;
+    private final static String VALUES_KEY = "values";
 
+    private final Schema schema;
     private EventStream event = null;
     private int eventMessageCount = 0;
 
     private int readCount = 0;
-    private Map<String, Value> message = Maps.newHashMap();
+    private List<Value> values;
 
-    public InForwardEventReader(Schema schema, TimestampParser timestampParser)
+    public InForwardEventReader(Schema schema)
     {
         this.schema = schema;
-        ImmutableMap.Builder<String, Column> builder = ImmutableMap.builder();
-        schema.getColumns().forEach(column -> builder.put(column.getName(), column));
-        this.columnMap = builder.build();
-        this.timestampParser = timestampParser;
     }
 
-    public Schema getSchema()
+    public void setEvent(EventStream event)
     {
-        return schema;
+        this.event = event;
+        this.eventMessageCount = event.getEntries().size();
     }
 
-    private Column getColumn(String columnName)
+    @Override
+    public Schema getSchema()
     {
-        return columnMap.get(columnName);
+        return schema;
     }
 
-    private Column getColumn(int columnIndex)
+    @Override
+    public boolean nextRecord()
     {
-        return getSchema().getColumn(columnIndex);
-    }
+        if (eventMessageCount <= readCount) {
+            return false;
+        }
 
-    private Value getValue(Column column)
-    {
-        return message.get(column.getName());
+        Optional<List<Value>> values = event.getEntries().get(readCount++).getRecord().entrySet()
+                .stream()
+                .filter(valueValueEntry -> valueValueEntry.getKey().asStringValue().asString().contentEquals(VALUES_KEY))
+                .map(valueValueEntry -> valueValueEntry.getValue().asArrayValue().list())
+                .findFirst();
+
+        this.values = values.orElse(Lists.newArrayListWithCapacity(schema.getColumnCount()));
+        return true;
     }
 
-    public void setEvent(EventStream event)
+    private Value getValue(int columnIndex)
     {
-        this.event = event;
-        this.eventMessageCount = event.getEntries().size();
+        return values.get(columnIndex);
     }
 
+    @Override
     public boolean isNull(Column column)
     {
-        return getValue(column).isNilValue();
+        return isNull(column.getIndex());
     }
 
+    @Override
     public boolean isNull(int columnIndex)
     {
-        return isNull(getColumn(columnIndex));
+        return getValue(columnIndex).isNilValue();
     }
 
+    @Override
     public boolean getBoolean(Column column)
     {
-        return getValue(column).asBooleanValue().getBoolean();
+        return getBoolean(column.getIndex());
     }
 
+    @Override
     public boolean getBoolean(int columnIndex)
     {
-        return getBoolean(getColumn(columnIndex));
+        return getValue(columnIndex).asBooleanValue().getBoolean();
     }
 
-    public long getLong(Column column)
+    @Override
+    public String getString(Column column)
     {
-        return getValue(column).asIntegerValue().asLong();
+        return getString(column.getIndex());
     }
 
-    public long getLong(int columnIndex)
+    @Override
+    public String getString(int columnIndex)
     {
-        return getLong(getColumn(columnIndex));
+        return getValue(columnIndex).asStringValue().asString();
     }
 
-    public double getDouble(Column column)
+    @Override
+    public long getLong(Column column)
     {
-        return getValue(column).asFloatValue().toDouble();
+        return getLong(column.getIndex());
     }
 
-    public double genDouble(int columnIndex)
+    @Override
+    public long getLong(int columnIndex)
     {
-        return getDouble(getColumn(columnIndex));
+        return getValue(columnIndex).asIntegerValue().asLong();
     }
 
-    public String getString(Column column)
+    @Override
+    public double getDouble(Column column)
     {
-        return getValue(column).toString();
+        return getDouble(column.getIndex());
     }
 
-    public String getString(int columnIndex)
+    @Override
+    public double getDouble(int columnIndex)
     {
-        return getString(getColumn(columnIndex));
+        return getValue(columnIndex).asFloatValue().toDouble();
     }
 
+    @Override
     public Timestamp getTimestamp(Column column)
     {
-        return timestampParser.parse(getString(column));
+        return getTimestamp(column.getIndex());
     }
 
+    @Override
     public Timestamp getTimestamp(int columnIndex)
     {
-        return getTimestamp(getColumn(columnIndex));
+        List<Value> seed = getValue(columnIndex).asArrayValue().list();
+        long epochSecond = seed.get(0).asIntegerValue().asLong();
+        long nanoAdjustment = seed.get(1).asIntegerValue().asLong();
+        return Timestamp.ofEpochSecond(epochSecond, nanoAdjustment);
     }
 
+    @Override
     public Value getJson(Column column)
     {
-        return jsonParser.parse(getString(column));
+        return getJson(column.getIndex());
     }
 
+    @Override
     public Value getJson(int columnIndex)
     {
-        return getJson(getColumn(columnIndex));
+        return getValue(columnIndex);
     }
 
-    public boolean nextMessage()
-    {
-        if (eventMessageCount <= readCount) {
-            return false;
-        }
-
-        ImmutableMap.Builder<String, Value> builder = ImmutableMap.builder();
-        event.getEntries().get(readCount++).getRecord().entrySet().forEach(
-                entry -> builder.put(entry.getKey().toString(), entry.getValue())
-        );
-        message = builder.build();
-        return true;
-    }
 }