embulk-filter-copy 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3044893452487bfd8e401d210931f2cd8c95b750
-  data.tar.gz: bcd788f6fcbf6f5e50c1867bbb85869fbe7781b1
+  metadata.gz: d2500f6ad78f1db21f25444a8454312d190d3229
+  data.tar.gz: 6b3f59c7329235aeaff13ee152a235617b282ab1
 SHA512:
-  metadata.gz: e7e68823705f1747c65bae165c6a573e5f3be6befb1b2ff03fedb95bac628350250405915445a247edb98cd6388dd3d73b57e631b45454bf8bdc44823c31d323
-  data.tar.gz: 0c16332e95084910ffe7f128abf68cac89e67c05151bf1bbbe86be666ab43e3ecb57e18c0e6725d17c143cf6a368796d033344d6e70d1ad3d250f8323b9e3099
+  metadata.gz: 6498b12ec28031b054e31cd14a35a37540942a23fcd6b0dd95149609afa0797a201badadc0ccddecb1baf49dfdfbfbd281dc5da9fa95d875bee69e25be0c90b8
+  data.tar.gz: 0f1552016b5e7417a69a868a73820c26451c7f7e2b13fb0dc773d031be8c70e4fbb660de74fcf8bf32fed9bf28c8adb8f2567379da6a47615284ad94f8db1a22

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.0.2 (2017-07-01)
+==================
+* Write a default values test.
+* Change EmbulkExecutorService to EmbulkExecutor/LocalThreadExecutor for #6
+
 0.0.1 (2017-06-07)
 ==================
-- first release.
+* first release.

data/README.md

@@ -34,6 +34,8 @@ filters:
 - This plugin has no test yet, so may have some bugs.
 - This plugin does not work on [embulk-executor-mapreduce](https://github.com/embulk/embulk-executor-mapreduce) yet.
 - This plugin uses lots of memory now, because embulk run twice.
+- Specification about error handling is not yet fixed.
+- If you have any problems or opinions, I'm glad if you raise Issue.
 
 ## Dependencies
 - https://github.com/okumin/influent

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.0.1"
+version = "0.0.2"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -27,7 +27,9 @@ dependencies {
     // This plugin use 1.1.0 for using the same msgpack version as embulk.
     // ref. https://github.com/komamitsu/fluency/compare/1.1.0...1.3.0
     compile  "org.komamitsu:fluency:1.1.0"
+
     testCompile "junit:junit:4.+"
+    testCompile "org.embulk:embulk-core:0.8.23:tests"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java

@@ -5,13 +5,13 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
+import org.embulk.filter.copy.executor.EmbulkExecutor;
 import org.embulk.filter.copy.forward.InForwardService;
 import org.embulk.filter.copy.forward.OutForwardEventBuilder;
 import org.embulk.filter.copy.forward.OutForwardService;
 import org.embulk.filter.copy.forward.OutForwardVisitor;
 import org.embulk.filter.copy.plugin.InternalForwardInputPlugin;
-import org.embulk.filter.copy.service.EmbulkExecutorService;
-import org.embulk.filter.copy.service.StandardColumnVisitor;
+import org.embulk.filter.copy.util.StandardColumnVisitor;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.Page;
@@ -45,7 +45,7 @@ public class CopyFilterPlugin
     }
 
     public interface PluginTask
-            extends Task, TimestampFormatter.Task,
+            extends Task, EmbulkExecutor.Task, TimestampFormatter.Task,
             OutForwardService.Task, InForwardService.Task
     {
         @Config("config")
@@ -72,15 +72,15 @@ public class CopyFilterPlugin
         return embulkRunConfig;
     }
 
-    private void withEmbulkRun(ConfigSource config, Runnable r)
+    private void withEmbulkRun(EmbulkExecutor executor, ConfigSource config, Runnable r)
     {
-        EmbulkExecutorService embulkExecutorService = new EmbulkExecutorService(Exec.getInjector());
-        embulkExecutorService.executeAsync(config);
+        executor.setup();
+        executor.executeAsync(config);
 
         r.run();
 
-        embulkExecutorService.waitExecutionFinished();
-        embulkExecutorService.shutdown();
+        executor.waitUntilExecutionFinished();
+        executor.shutdown();
     }
 
     @Override
@@ -89,11 +89,11 @@ public class CopyFilterPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
         ConfigSource embulkRunConfig = configure(task, inputSchema);
+        EmbulkExecutor embulkExecutor = EmbulkExecutor.buildExecutor(task);
 
-        withEmbulkRun(embulkRunConfig, () ->
-        {
-            Schema outputSchema = inputSchema;
-            control.run(task.dump(), outputSchema);
+        withEmbulkRun(embulkExecutor, embulkRunConfig, () -> {
+            Schema outputSchema1 = inputSchema;
+            control.run(task.dump(), outputSchema1);
             OutForwardService.sendShutdownMessage(task);
         });
     }

data/src/main/java/org/embulk/filter/copy/executor/EmbulkExecutor.java

@@ -0,0 +1,105 @@
+package org.embulk.filter.copy.executor;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.google.common.base.Joiner;
+import com.google.inject.Injector;
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.guice.LifeCycleInjector;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Locale;
+
+public abstract class EmbulkExecutor
+{
+    static final Logger logger = Exec.getLogger(EmbulkExecutor.class);
+
+    public interface ExecutorTask
+            extends org.embulk.config.Task
+    {
+        @Config("\"type\"")
+        @ConfigDefault("\"local_thread\"")
+        ExecutorType getType();
+
+        enum ExecutorType
+        {
+            LOCAL_THREAD;
+
+            @JsonValue
+            @Override
+            public String toString()
+            {
+                return name().toLowerCase(Locale.ENGLISH);
+            }
+
+            @JsonCreator
+            public static ExecutorType fromString(String value)
+            {
+                switch (value) {
+                    case "local_thread":
+                        return LOCAL_THREAD;
+                    default:
+                        String values = Joiner.on(", ").join(values()).toLowerCase(Locale.ENGLISH);
+                        throw new ConfigException(String.format("Unknown executor type '%s'. Supported executor types are %s", value, values));
+                }
+            }
+        }
+
+    }
+
+    public interface Task
+        extends org.embulk.config.Task
+    {
+        @Config("executor")
+        @ConfigDefault("{}")
+        ExecutorTask getExecutorTask();
+    }
+
+    public static EmbulkExecutor buildExecutor(Task task)
+    {
+        switch (task.getExecutorTask().getType()) {
+            case LOCAL_THREAD:
+                return new LocalThreadExecutor(task.getExecutorTask());
+            default:
+                String values = Joiner.on(", ").join(ExecutorTask.ExecutorType.values()).toLowerCase(Locale.ENGLISH);
+                throw new ConfigException(String.format("Unknown executor type '%s'. Supported executor types are %s", task.getExecutorTask().getType(), values));
+        }
+    }
+
+    final ExecutorTask task;
+    final Injector injector;
+
+    EmbulkExecutor(ExecutorTask task)
+    {
+        this.task = task;
+        this.injector = Exec.getInjector();
+    }
+
+    public abstract void setup();
+
+    public abstract void executeAsync(ConfigSource config);
+
+    public abstract void waitUntilExecutionFinished();
+
+    public abstract void shutdown();
+
+    EmbulkEmbed newEmbulkEmbed()
+    {
+        try {
+            Constructor<EmbulkEmbed> constructor = EmbulkEmbed.class
+                    .getDeclaredConstructor(ConfigSource.class, LifeCycleInjector.class);
+            constructor.setAccessible(true);
+            return constructor.newInstance(null, injector);
+        }
+        catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
+            throw new ConfigException(e);
+        }
+    }
+}

data/src/main/java/org/embulk/filter/copy/{service/EmbulkExecutorService.java → executor/LocalThreadExecutor.java}

@@ -1,39 +1,30 @@
-package org.embulk.filter.copy.service;
+package org.embulk.filter.copy.executor;
 
 import com.google.common.util.concurrent.FutureCallback;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.ListeningExecutorService;
 import com.google.common.util.concurrent.MoreExecutors;
-import com.google.inject.Injector;
-import org.embulk.EmbulkEmbed;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.exec.ExecutionResult;
 import org.embulk.filter.copy.util.ElapsedTime;
-import org.embulk.guice.LifeCycleInjector;
-import org.embulk.spi.Exec;
-import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
 
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
 import java.util.concurrent.Callable;
 import java.util.concurrent.Executors;
 
-public class EmbulkExecutorService
+public class LocalThreadExecutor
+    extends EmbulkExecutor
 {
-    private final static String THREAD_NAME = "embulk executor service";
+    private static final String THREAD_NAME = "embulk executor service";
     private static final int NUM_THREADS = 1;
-    private final static Logger logger = Exec.getLogger(EmbulkExecutorService.class);
-    private final Injector injector;
     private final ListeningExecutorService es;
     private ListenableFuture<ExecutionResult> future;
 
-    public EmbulkExecutorService(Injector injector)
+    LocalThreadExecutor(ExecutorTask task)
     {
-        this.injector = injector;
+        super(task);
         this.es = MoreExecutors.listeningDecorator(
                 Executors.newFixedThreadPool(
                         NUM_THREADS,
@@ -41,6 +32,12 @@ public class EmbulkExecutorService
                 ));
     }
 
+    @Override
+    public void setup()
+    {
+    }
+
+    @Override
     public void executeAsync(ConfigSource config)
     {
         logger.debug("execute with this config: {}", config);
@@ -51,40 +48,28 @@ public class EmbulkExecutorService
         Futures.addCallback(future, resultFutureCallback());
     }
 
-    public void shutdown()
-    {
-        ElapsedTime.info(
-                logger,
-                "embulk executor service shutdown",
-                es::shutdown);
-    }
-
-    public void waitExecutionFinished()
+    @Override
+    public void waitUntilExecutionFinished()
     {
         if (future == null) {
             throw new NullPointerException();
         }
-
         ElapsedTime.debugUntil(() -> future.isDone() || future.isCancelled(),
                 logger, "embulk executor", 3000L);
     }
 
-    private Callable<ExecutionResult> embulkRun(ConfigSource config)
+    @Override
+    public void shutdown()
     {
-        return () -> newEmbulkEmbed(injector).run(config);
+        ElapsedTime.info(
+                logger,
+                "embulk executor service shutdown",
+                es::shutdown);
     }
 
-    private EmbulkEmbed newEmbulkEmbed(Injector injector)
+    private Callable<ExecutionResult> embulkRun(ConfigSource config)
     {
-        try {
-            Constructor<EmbulkEmbed> constructor = EmbulkEmbed.class
-                    .getDeclaredConstructor(ConfigSource.class, LifeCycleInjector.class);
-            constructor.setAccessible(true);
-            return constructor.newInstance(Exec.newConfigSource(), injector);
-        }
-        catch (InstantiationException | IllegalAccessException | NoSuchMethodException | InvocationTargetException e) {
-            throw new ConfigException(e);
-        }
+        return () -> newEmbulkEmbed().run(config);
     }
 
     private FutureCallback<ExecutionResult> resultFutureCallback()

data/src/main/java/org/embulk/filter/copy/{service → util}/StandardColumnVisitor.java

@@ -1,4 +1,4 @@
-package org.embulk.filter.copy.service;
+package org.embulk.filter.copy.util;
 
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;

data/src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java

@@ -1,5 +1,113 @@
 package org.embulk.filter.copy;
 
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigLoader;
+import org.embulk.config.ConfigSource;
+import org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Schema;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+import org.joda.time.DateTimeZone;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static com.google.common.base.Optional.absent;
+import static org.embulk.filter.copy.CopyFilterPlugin.EmbulkConfig;
+import static org.embulk.filter.copy.CopyFilterPlugin.PluginTask;
+import static org.embulk.filter.copy.executor.EmbulkExecutor.ExecutorTask.ExecutorType.LOCAL_THREAD;
+import static org.embulk.filter.copy.forward.InForwardService.InForwardTask;
+import static org.embulk.filter.copy.forward.OutForwardService.OutForwardTask;
+import static org.junit.Assert.assertEquals;
+
 public class TestCopyFilterPlugin
 {
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Rule
+    public ExpectedException exception = ExpectedException.none();
+
+    private Schema schema;
+    private CopyFilterPlugin plugin;
+
+    private static Schema schema(Object... nameAndTypes)
+    {
+        Schema.Builder builder = Schema.builder();
+        for (int i = 0; i < nameAndTypes.length; i += 2) {
+            builder.add((String) nameAndTypes[i], (Type) nameAndTypes[i + 1]);
+        }
+        return builder.build();
+    }
+
+    private static ConfigSource fromYaml(String yaml)
+    {
+        ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
+        return loader.fromYamlString(yaml);
+    }
+
+    @Before
+    public void createResources()
+    {
+        schema = schema("_c0", Types.STRING, "_c1", Types.STRING); // default schema
+    }
+
+    @Test
+    public void testPluginTaskDefault()
+    {
+        String yaml = String.join("\n",
+                "type: copy",
+                "config:",
+                "  out:",
+                "    type: stdout"
+        );
+
+        ConfigSource config = fromYaml(yaml);
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // EmbulkConfig
+        EmbulkConfig embulkConfig = task.getConfig();
+        assertEquals(Exec.newConfigSource(), embulkConfig.getExecConfig());
+        assertEquals(Lists.<ConfigSource>newArrayList(), embulkConfig.getFilterConfig());
+
+        // EmbulkExecutor
+        ExecutorTask executorTask = task.getExecutorTask();
+        assertEquals(LOCAL_THREAD, executorTask.getType());
+
+        // TimestampFormatter
+        assertEquals(DateTimeZone.UTC, task.getDefaultTimeZone());
+        assertEquals("%Y-%m-%d %H:%M:%S.%6N %z", task.getDefaultTimestampFormat());
+
+        // ForwardBaseTask
+        assertEquals("message", task.getMessageTag());
+        assertEquals("shutdown", task.getShutdownTag());
+
+        // InForwardService
+        InForwardTask inForward = task.getInForwardTask();
+        assertEquals(24224, inForward.getPort());
+        assertEquals(absent(), inForward.getChunkSizeLimit());
+        assertEquals(absent(), inForward.getBacklog());
+        assertEquals(absent(), inForward.getSendBufferSize());
+        assertEquals(absent(), inForward.getReceiveBufferSize());
+        assertEquals(absent(), inForward.getKeepAliveEnabled());
+        assertEquals(absent(), inForward.getTcpNoDelayEnabled());
+
+        // OutForwardService
+        OutForwardTask outForward = task.getOutForwardTask();
+        assertEquals("localhost", outForward.getHost());
+        assertEquals(24224, outForward.getPort());
+        assertEquals(absent(), outForward.getMaxBufferSize());
+        assertEquals(absent(), outForward.getBufferChunkInitialSize());
+        assertEquals(absent(), outForward.getBufferChunkRetentionSize());
+        assertEquals(absent(), outForward.getFlushIntervalMillis());
+        assertEquals(absent(), outForward.getSenderMaxRetryCount());
+        assertEquals(absent(), outForward.getAckResponseMode());
+        assertEquals(absent(), outForward.getFileBackupDir());
+        assertEquals(absent(), outForward.getWaitUntilBufferFlushed());
+        assertEquals(absent(), outForward.getWaitUntilFlusherTerminated());
+    }
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-filter-copy
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-06 00:00:00.000000000 Z
+date: 2017-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,6 +61,8 @@ files:
 - lib/embulk/filter/copy.rb
 - settings.gradle
 - src/main/java/org/embulk/filter/copy/CopyFilterPlugin.java
+- src/main/java/org/embulk/filter/copy/executor/EmbulkExecutor.java
+- src/main/java/org/embulk/filter/copy/executor/LocalThreadExecutor.java
 - src/main/java/org/embulk/filter/copy/forward/ForwardBaseTask.java
 - src/main/java/org/embulk/filter/copy/forward/InForwardEventReader.java
 - src/main/java/org/embulk/filter/copy/forward/InForwardService.java
@@ -69,12 +71,11 @@ files:
 - src/main/java/org/embulk/filter/copy/forward/OutForwardService.java
 - src/main/java/org/embulk/filter/copy/forward/OutForwardVisitor.java
 - src/main/java/org/embulk/filter/copy/plugin/InternalForwardInputPlugin.java
-- src/main/java/org/embulk/filter/copy/service/EmbulkExecutorService.java
-- src/main/java/org/embulk/filter/copy/service/StandardColumnVisitor.java
 - src/main/java/org/embulk/filter/copy/util/ElapsedTime.java
+- src/main/java/org/embulk/filter/copy/util/StandardColumnVisitor.java
 - src/test/java/org/embulk/filter/copy/TestCopyFilterPlugin.java
 - src/test/java/org/embulk/filter/copy/plugin/TestInternalForwardInputPlugin.java
-- classpath/embulk-filter-copy-0.0.1.jar
+- classpath/embulk-filter-copy-0.0.2.jar
 - classpath/fluency-1.1.0.jar
 - classpath/influent-java-0.2.0.jar
 - classpath/influent-transport-0.2.0.jar