embulk-filter-copy 0.0.2 → 0.1.0

data/src/main/java/org/embulk/filter/copy/runner/AsyncEmbulkRunnerService.java

@@ -0,0 +1,32 @@
+package org.embulk.filter.copy.runner;
+
+import com.google.common.util.concurrent.AbstractExecutionThreadService;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class AsyncEmbulkRunnerService
+        extends AbstractExecutionThreadService
+{
+    private static final Logger logger = Exec.getLogger(AsyncEmbulkRunnerService.class);
+    private final EmbulkRunner runner;
+
+    public AsyncEmbulkRunnerService(EmbulkRunner runner)
+    {
+        this.runner = runner;
+    }
+
+    @Override
+    protected void run()
+            throws Exception
+    {
+        ExecutionResult result = runner.run();
+        if (result.isSkipped()) {
+            logger.warn("ExecutionResult: EmbulkRunner.run is skipped");
+        }
+        if (!result.getIgnoredExceptions().isEmpty()) {
+            result.getIgnoredExceptions().forEach(e -> logger.warn("Ignored Error is found", e));
+        }
+        logger.debug("Execution Result: {}", result);
+    }
+}

data/src/main/java/org/embulk/filter/copy/runner/EmbulkRunner.java

@@ -0,0 +1,124 @@
+package org.embulk.filter.copy.runner;
+
+import com.google.common.collect.Lists;
+import com.google.inject.Injector;
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.exec.ExecutionResult;
+import org.embulk.filter.copy.spi.ElapsedTime;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.List;
+import java.util.Optional;
+
+public class EmbulkRunner
+{
+    private static final Logger logger = Exec.getLogger(EmbulkRunner.class);
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    public static class Builder
+    {
+        private ConfigSource execConfig;
+        private ConfigSource inputConfig;
+        private List<ConfigSource> filtersConfig;
+        private ConfigSource outputConfig;
+
+        public Builder()
+        {
+        }
+
+        public Builder execConfig(ConfigSource execConfig)
+        {
+            this.execConfig = execConfig;
+            return this;
+        }
+
+        public Builder inputConfig(ConfigSource inputConfig)
+        {
+            this.inputConfig = inputConfig;
+            return this;
+        }
+
+        public Builder filterConfig(List<ConfigSource> filtersConfig)
+        {
+            this.filtersConfig = filtersConfig;
+            return this;
+        }
+
+        public Builder outputConfig(ConfigSource outputConfig)
+        {
+            this.outputConfig = outputConfig;
+            return this;
+        }
+
+        public EmbulkRunner build()
+        {
+            return new EmbulkRunner(buildConfig());
+        }
+
+        protected ConfigSource buildConfig()
+        {
+            ConfigSource config = Exec.newConfigSource();
+
+            config.set("exec", Optional.ofNullable(execConfig).orElse(Exec.newConfigSource()));
+            config.set("in", Optional.ofNullable(inputConfig).orElseThrow(() -> new ConfigException("in: is null.")));
+            config.set("filters", Optional.ofNullable(filtersConfig).orElse(Lists.newArrayList()));
+            config.set("out", Optional.ofNullable(outputConfig).orElseThrow(() -> new ConfigException("out: is null.")));
+
+            return config;
+        }
+    }
+
+    private final ConfigSource config;
+
+    EmbulkRunner(ConfigSource config)
+    {
+        this.config = config;
+    }
+
+    private EmbulkEmbed newEmbulkEmbed()
+    {
+        try {
+            Constructor<EmbulkEmbed> constructor = EmbulkEmbed.class
+                    .getDeclaredConstructor(ConfigSource.class, Injector.class);
+            constructor.setAccessible(true);
+            return constructor.newInstance(null, Exec.getInjector());
+        }
+        catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+            throw new ConfigException(e);
+        }
+    }
+
+    public ExecutionResult run()
+    {
+        // TODO: expose config without secret configurations.
+        return ElapsedTime.measure(new ElapsedTime.Measurable<ExecutionResult>()
+        {
+            @Override
+            public void onStart()
+            {
+                logger.info("Start: Embulk Run");
+            }
+
+            @Override
+            public void onFinished(long elapsedMillis)
+            {
+                logger.info("Finished: Embulk Run (Elapsed: {} ms)", elapsedMillis);
+            }
+
+            @Override
+            public ExecutionResult run()
+            {
+                return newEmbulkEmbed().run(config);
+            }
+        });
+    }
+}

data/src/main/java/org/embulk/filter/copy/spi/DataBuilder.java

@@ -0,0 +1,41 @@
+package org.embulk.filter.copy.spi;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public interface DataBuilder
+{
+    Schema getSchema();
+
+    void addRecord();
+
+    void setNull(Column column);
+
+    void setNull(int columnIndex);
+
+    void setBoolean(Column column, boolean v);
+
+    void setBoolean(int columnIndex, boolean v);
+
+    void setString(Column column, String v);
+
+    void setString(int columnIndex, String v);
+
+    void setLong(Column column, long v);
+
+    void setLong(int columnIndex, long v);
+
+    void setDouble(Column column, double v);
+
+    void setDouble(int columnIndex, double v);
+
+    void setTimestamp(Column column, Timestamp v);
+
+    void setTimestamp(int columnIndex, Timestamp v);
+
+    void setJson(Column column, Value v);
+
+    void setJson(int columnIndex, Value v);
+}

data/src/main/java/org/embulk/filter/copy/spi/DataReader.java

@@ -0,0 +1,41 @@
+package org.embulk.filter.copy.spi;
+
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.time.Timestamp;
+import org.msgpack.value.Value;
+
+public interface DataReader
+{
+    Schema getSchema();
+
+    boolean nextRecord();
+
+    boolean isNull(Column column);
+
+    boolean isNull(int columnIndex);
+
+    boolean getBoolean(Column column);
+
+    boolean getBoolean(int columnIndex);
+
+    String getString(Column column);
+
+    String getString(int columnIndex);
+
+    long getLong(Column column);
+
+    long getLong(int columnIndex);
+
+    double getDouble(Column column);
+
+    double getDouble(int columnIndex);
+
+    Timestamp getTimestamp(Column column);
+
+    Timestamp getTimestamp(int columnIndex);
+
+    Value getJson(Column column);
+
+    Value getJson(int columnIndex);
+}

data/src/main/java/org/embulk/filter/copy/spi/ElapsedTime.java

@@ -0,0 +1,81 @@
+package org.embulk.filter.copy.spi;
+
+public class ElapsedTime
+{
+    private final static long DEFAULT_POLLING_INTERVAL = 1_000; // ms
+
+    private static long getNow()
+    {
+        return System.currentTimeMillis();
+    }
+
+    private static long getElapsed(long start)
+    {
+        return System.currentTimeMillis() - start;
+    }
+
+    private static void waitUntilNextPolling(long pollingInterval)
+    {
+        try {
+            Thread.sleep(pollingInterval);
+        }
+        catch (InterruptedException e) {
+            // Do Nothing
+        }
+    }
+
+    public interface Measurable<T>
+    {
+        T run();
+
+        void onStart();
+
+        void onFinished(long elapsedMillis);
+    }
+
+    public interface Pollable<T>
+    {
+        boolean poll();
+
+        T getResult();
+
+        void onStart();
+
+        void onWaiting(long elapsedMillis);
+
+        void onFinished(long elapsedMillis);
+    }
+
+    public static <T> T measure(Measurable<T> measurable)
+    {
+        long start = getNow();
+        measurable.onStart();
+        try {
+            return measurable.run();
+        }
+        finally {
+            measurable.onFinished(getElapsed(start));
+        }
+    }
+
+    public static <T> T measureWithPolling(long pollingInterval, Pollable<T> pollable)
+    {
+        long start = getNow();
+        pollable.onStart();
+        while (!pollable.poll()) {
+            pollable.onWaiting(getElapsed(start));
+            waitUntilNextPolling(pollingInterval);
+        }
+        try {
+            return pollable.getResult();
+        }
+        finally {
+            pollable.onFinished(getElapsed(start));
+        }
+    }
+
+    public static <T> T measureWithPolling(Pollable<T> pollable)
+    {
+        return measureWithPolling(DEFAULT_POLLING_INTERVAL, pollable);
+    }
+}

data/src/main/java/org/embulk/filter/copy/spi/PageBuilder.java

@@ -0,0 +1,16 @@
+package org.embulk.filter.copy.spi;
+
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.filter.copy.spi.DataBuilder;
+
+public class PageBuilder
+        extends org.embulk.spi.PageBuilder
+        implements DataBuilder
+{
+    public PageBuilder(BufferAllocator allocator, Schema schema, PageOutput output)
+    {
+        super(allocator, schema, output);
+    }
+}

data/src/main/java/org/embulk/filter/copy/spi/PageReader.java

@@ -0,0 +1,13 @@
+package org.embulk.filter.copy.spi;
+
+import org.embulk.spi.Schema;
+
+public class PageReader
+        extends org.embulk.spi.PageReader
+        implements DataReader
+{
+    public PageReader(Schema schema)
+    {
+        super(schema);
+    }
+}

data/src/main/java/org/embulk/filter/copy/{util → spi}/StandardColumnVisitor.java

@@ -1,17 +1,17 @@
-package org.embulk.filter.copy.util;
+package org.embulk.filter.copy.spi;
 
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageReader;
+import org.embulk.filter.copy.spi.DataBuilder;
+import org.embulk.filter.copy.spi.DataReader;
 
 public class StandardColumnVisitor
     implements ColumnVisitor
 {
-    private final PageReader reader;
-    private final PageBuilder builder;
+    private final DataReader reader;
+    private final DataBuilder builder;
 
-    public StandardColumnVisitor(PageReader reader, PageBuilder builder)
+    public StandardColumnVisitor(DataReader reader, DataBuilder builder)
     {
         this.reader = reader;
         this.builder = builder;

data/src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java

@@ -4,12 +4,10 @@ import com.google.common.collect.Lists;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigLoader;
 import org.embulk.config.ConfigSource;
-import org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Schema;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.Types;
-import org.joda.time.DateTimeZone;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
@@ -18,7 +16,6 @@ import org.junit.rules.ExpectedException;
 import static com.google.common.base.Optional.absent;
 import static org.embulk.filter.copy.CopyFilterPlugin.EmbulkConfig;
 import static org.embulk.filter.copy.CopyFilterPlugin.PluginTask;
-import static org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask.ExecutorType.LOCAL_THREAD;
 import static org.embulk.filter.copy.forward.InForwardService.InForwardTask;
 import static org.embulk.filter.copy.forward.OutForwardService.OutForwardTask;
 import static org.junit.Assert.assertEquals;
@@ -73,14 +70,6 @@ public class TestCopyFilterPlugin
         assertEquals(Exec.newConfigSource(), embulkConfig.getExecConfig());
         assertEquals(Lists.<ConfigSource>newArrayList(), embulkConfig.getFilterConfig());
 
-        // EmbulkExecutor
-        ExecutorTask executorTask = task.getExecutorTask();
-        assertEquals(LOCAL_THREAD, executorTask.getType());
-
-        // TimestampFormatter
-        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
-        assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
-
         // ForwardBaseTask
         assertEquals("message", task.getMessageTag());
         assertEquals("shutdown", task.getShutdownTag());

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-copy
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.0
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-01 00:00:00.000000000 Z
+date: 2019-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,30 +61,38 @@ files:
 - lib/embulk/filter/copy.rb
 - settings.gradle
 - src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java
-- src/main/java/org/embulk/filter/copy/executor/EmbulkExecutor.java
-- src/main/java/org/embulk/filter/copy/executor/LocalThreadExecutor.java
-- src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java
+- src/main/java/org/embulk/filter/copy/forward/ForwardParentTask.java
 - src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java
 - src/main/java/org/embulk/filter/copy/forward/InForwardService.java
-- src/main/java/org/embulk/filter/copy/forward/InForwardVisitor.java
 - src/main/java/org/embulk/filter/copy/forward/OutForwardEventBuilder.java
 - src/main/java/org/embulk/filter/copy/forward/OutForwardService.java
-- src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java
 - src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java
-- src/main/java/org/embulk/filter/copy/util/ElapsedTime.java
-- src/main/java/org/embulk/filter/copy/util/StandardColumnVisitor.java
+- src/main/java/org/embulk/filter/copy/runner/AsyncEmbulkRunnerService.java
+- src/main/java/org/embulk/filter/copy/runner/EmbulkRunner.java
+- src/main/java/org/embulk/filter/copy/spi/DataBuilder.java
+- src/main/java/org/embulk/filter/copy/spi/DataReader.java
+- src/main/java/org/embulk/filter/copy/spi/ElapsedTime.java
+- src/main/java/org/embulk/filter/copy/spi/PageBuilder.java
+- src/main/java/org/embulk/filter/copy/spi/PageReader.java
+- src/main/java/org/embulk/filter/copy/spi/StandardColumnVisitor.java
 - src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java
 - src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java
-- classpath/embulk-filter-copy-0.0.2.jar
+- classpath/jsr305-1.3.9.jar
+- classpath/error_prone_annotations-2.0.18.jar
+- classpath/jackson-databind-2.7.1.jar
+- classpath/jackson-core-2.7.1.jar
+- classpath/embulk-filter-copy-0.1.0.jar
+- classpath/slf4j-api-1.7.25.jar
+- classpath/phi-accural-failure-detector-0.0.4.jar
+- classpath/influent-transport-0.3.0.jar
 - classpath/fluency-1.1.0.jar
-- classpath/influent-java-0.2.0.jar
-- classpath/influent-transport-0.2.0.jar
+- classpath/influent-java-0.3.0.jar
+- classpath/msgpack-core-0.8.13.jar
+- classpath/guava-23.0.jar
 - classpath/jackson-annotations-2.7.0.jar
-- classpath/jackson-core-2.7.1.jar
-- classpath/jackson-databind-2.7.1.jar
+- classpath/j2objc-annotations-1.1.jar
+- classpath/animal-sniffer-annotations-1.14.jar
 - classpath/jackson-dataformat-msgpack-0.8.11.jar
-- classpath/phi-accural-failure-detector-0.0.4.jar
-- classpath/slf4j-api-1.7.22.jar
 homepage: https://github.com/civitaspo/embulk-filter-copy
 licenses:
 - MIT