embulk-output-multi 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb52fdd126c8c9e752f186479dfdd1e8bb60560c
-  data.tar.gz: fd2f7fa9b9ee60bd2d7aaaa598b9ade60c611abf
+  metadata.gz: 40d2c979fbce0853ed920cae18f0f867284e9491
+  data.tar.gz: e02bef302e6b8061f9029dc4d7077a00b7563623
 SHA512:
-  metadata.gz: 4c8cd6915a54439599fc5a6f6373a6b72c2e678ecfa3ccf02d406a3342f2358b47bfe8b8dbe6661f74a59d622eda60fdd122bca2d5a9b8b95d0eb2d3d4cf6964
-  data.tar.gz: 16179171d941d670370d374cda08d683fa36e4553422b313b0c1e1db4c6649525c66ef6a466a7a23505cac98c9a086f074ec5ca153d233e34c906e322da51d00
+  metadata.gz: ba665f2f4108d65c1cf48ad5a94789e6d719443bb0e8bc7f579f05357407918ab275b41cd55aa96af0ae0854e572427f55039a8502e332c8ab22d6707d6118f9
+  data.tar.gz: 7fa76fd0c1a8c06e89b2b0eff90e6b16e00985db269444a37812da6af0f40e3a74a748b00c7e3dfc675b35f7c60875e5f57efd87bf6963eacf57f05cc04a9e79

data/README.md

@@ -18,7 +18,6 @@ This plugin can copies an output to multiple destinations.
 ## Configuration
 
 - **outputs**: Configuration of output plugins (array, required)
-- **stop_on_failed_output**: Set true if you want to immediately stop outputting of all plugins with a plugin's error (boolean, default "false")
 
 ## Example
 

data/gradle.properties

@@ -1 +1 @@
-version=0.2.0
+version=0.2.1

data/src/main/java/org/embulk/output/multi/{RunControlTask.java → AsyncRunControl.java}

@@ -15,7 +15,7 @@ import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
-class RunControlTask implements Callable<List<TaskReport>> {
+class AsyncRunControl {
     private static final String THREAD_NAME_FORMAT = "multi-run-control-%d";
     private final MultiOutputPlugin.PluginTask task;
     private final OutputPlugin.Control control;
@@ -24,7 +24,11 @@ class RunControlTask implements Callable<List<TaskReport>> {
     private final ExecutorService executorService;
     private Future<List<TaskReport>> result;
 
-    RunControlTask(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
+    static AsyncRunControl start(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
+        return new AsyncRunControl(task, control).start();
+    }
+
+    private AsyncRunControl(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
         this.task = task;
         this.control = control;
         this.latch = new CountDownLatch(task.getOutputConfigs().size());
@@ -32,18 +36,6 @@ class RunControlTask implements Callable<List<TaskReport>> {
         this.executorService = Executors.newSingleThreadExecutor(
                 new ThreadFactoryBuilder().setNameFormat(THREAD_NAME_FORMAT).build()
         );
-        this.result = executorService.submit(this);
-    }
-
-    @Override
-    public List<TaskReport> call() throws Exception {
-        latch.await();
-        List<TaskSource> taskSources = new ArrayList<>(this.taskSources.length());
-        for (int i = 0; i < this.taskSources.length(); i++) {
-            taskSources.add(this.taskSources.get(i));
-        }
-        task.setTaskSources(taskSources);
-        return control.run(task.dump());
     }
 
     void cancel() {
@@ -62,4 +54,22 @@ class RunControlTask implements Callable<List<TaskReport>> {
             executorService.shutdown();
         }
     }
+
+    private AsyncRunControl start() {
+        this.result = executorService.submit(new RunControl());
+        return this;
+    }
+
+    private class RunControl implements Callable<List<TaskReport>> {
+        @Override
+        public List<TaskReport> call() throws Exception {
+            latch.await();
+            List<TaskSource> taskSources = new ArrayList<>(AsyncRunControl.this.taskSources.length());
+            for (int i = 0; i < AsyncRunControl.this.taskSources.length(); i++) {
+                taskSources.add(AsyncRunControl.this.taskSources.get(i));
+            }
+            task.setTaskSources(taskSources);
+            return control.run(task.dump());
+        }
+    }
 }

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -48,9 +48,9 @@ public class MultiOutputPlugin implements OutputPlugin {
             throw new ConfigException("'outputs' must have more than than or equals to 1 element.");
         }
         final ExecSession session = Exec.session();
-        final RunControlTask controlTask = new RunControlTask(task, control);
+        final AsyncRunControl runControl = AsyncRunControl.start(task, control);
         return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
-                delegate.transaction(schema, taskCount, controlTask)
+                delegate.transaction(schema, taskCount, runControl)
         ));
     }
 
@@ -58,9 +58,9 @@ public class MultiOutputPlugin implements OutputPlugin {
     public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final ExecSession session = Exec.session();
-        final RunControlTask controlTask = new RunControlTask(task, control);
+        final AsyncRunControl runControl = AsyncRunControl.start(task, control);
         return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
-                delegate.resume(schema, taskCount, controlTask)
+                delegate.resume(schema, taskCount, runControl)
         ));
     }
 

data/src/main/java/org/embulk/output/multi/OutputPluginDelegate.java

@@ -15,6 +15,7 @@ import org.slf4j.LoggerFactory;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
@@ -46,17 +47,17 @@ class OutputPluginDelegate {
         );
     }
 
-    Future<ConfigDiff> transaction(Schema schema, int taskCount, RunControlTask controlTask) {
+    Future<ConfigDiff> transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
         return executorService.submit(() -> {
             try {
                 LOGGER.debug("Run #transaction for {}", getPluginCode());
-                return plugin.transaction(config, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+                return plugin.transaction(config, schema, taskCount, new Control(pluginIndex, runControl));
             } catch (CancellationException e) {
                 LOGGER.error("Canceled #transaction for {} by other plugin's error", getPluginCode());
                 throw e;
             } catch (Exception e) {
                 LOGGER.error("Transaction for {} failed.", getPluginCode(), e);
-                controlTask.cancel();
+                runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
@@ -64,17 +65,17 @@ class OutputPluginDelegate {
         });
     }
 
-    Future<ConfigDiff> resume(Schema schema, int taskCount, RunControlTask controlTask) {
+    Future<ConfigDiff> resume(Schema schema, int taskCount, AsyncRunControl runControl) {
         return executorService.submit(() -> {
             try {
                 LOGGER.debug("Run #resume for {}", getPluginCode());
-                return plugin.resume(taskSource, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+                return plugin.resume(taskSource, schema, taskCount, new Control(pluginIndex, runControl));
             } catch (CancellationException e) {
                 LOGGER.error("Canceled #resume for {} by other plugin's error", getPluginCode());
                 throw e;
             } catch (Exception e) {
                 LOGGER.error("Resume for {} failed.", getPluginCode(), e);
-                controlTask.cancel();
+                runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
@@ -108,4 +109,35 @@ class OutputPluginDelegate {
     private static String generatePluginCode(PluginType type, int pluginIndex) {
         return String.format("%s-%d", type.getName(), pluginIndex);
     }
+
+    private static class Control implements OutputPlugin.Control {
+        private final int pluginIndex;
+        private final AsyncRunControl runControl;
+
+        Control(int index, AsyncRunControl runControl) {
+            this.pluginIndex = index;
+            this.runControl = runControl;
+        }
+
+        @Override
+        public List<TaskReport> run(TaskSource taskSource) {
+            runControl.addTaskSource(pluginIndex, taskSource);
+            List<TaskReport> reports;
+            try {
+                reports = runControl.waitAndGetResult();
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
+                throw new RuntimeException(e);
+            } catch (ExecutionException e) {
+                throw new RuntimeException(e.getCause());
+            }
+
+            final List<TaskReport> result = new ArrayList<>();
+            for (TaskReport taskReport : reports) {
+                final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+                result.add(report);
+            }
+            return result;
+        }
+    }
 }

data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java

@@ -82,8 +82,7 @@ class TransactionalPageOutputDelegate {
         @Override
         public TaskReport call() throws InterruptedException {
             while (true) {
-                final Supplier<Object> task = taskQueue.take();
-                final Object result = task.get();
+                final Object result = taskQueue.take().get();
                 if (result instanceof TaskReport) {
                     return (TaskReport) result;
                 }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-multi
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Shinichi ISHIMURA
@@ -50,7 +50,7 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-output-multi-0.2.0.jar
+- classpath/embulk-output-multi-0.2.1.jar
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile
@@ -60,11 +60,10 @@ files:
 - gradlew.bat
 - lib/embulk/output/multi.rb
 - settings.gradle
-- src/main/java/org/embulk/output/multi/ControlDelegate.java
+- src/main/java/org/embulk/output/multi/AsyncRunControl.java
 - src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
 - src/main/java/org/embulk/output/multi/OutputPluginDelegate.java
 - src/main/java/org/embulk/output/multi/PluginExecutionException.java
-- src/main/java/org/embulk/output/multi/RunControlTask.java
 - src/main/java/org/embulk/output/multi/TaskReports.java
 - src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java
 - src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

data/src/main/java/org/embulk/output/multi/ControlDelegate.java

@@ -1,40 +0,0 @@
-package org.embulk.output.multi;
-
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.OutputPlugin;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-class ControlDelegate implements OutputPlugin.Control {
-    private final int pluginIndex;
-    private final RunControlTask controlTask;
-
-    ControlDelegate(int index, RunControlTask controlTask) {
-        this.pluginIndex = index;
-        this.controlTask = controlTask;
-    }
-
-    @Override
-    public List<TaskReport> run(TaskSource taskSource) {
-        controlTask.addTaskSource(pluginIndex, taskSource);
-        List<TaskReport> reports;
-        try {
-            reports = controlTask.waitAndGetResult();
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new RuntimeException(e);
-        } catch (ExecutionException e) {
-            throw new RuntimeException(e.getCause());
-        }
-
-        final List<TaskReport> result = new ArrayList<>();
-        for (TaskReport taskReport : reports) {
-            final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
-            result.add(report);
-        }
-        return result;
-    }
-}