embulk-output-multi 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd9a500d9ed46b10465db1c1521c60324a90526b
-  data.tar.gz: fc26433a4aabf02ad3e474c503284801e816c0d8
+  metadata.gz: cb52fdd126c8c9e752f186479dfdd1e8bb60560c
+  data.tar.gz: fd2f7fa9b9ee60bd2d7aaaa598b9ade60c611abf
 SHA512:
-  metadata.gz: f65a6b68738a9af5638550666c3340d30f8e75fcbe7b142843a977492c98179c4d7f5c1dd875041771608e7e72c4f57113432ca55695b5f07bd378a191446cdd
-  data.tar.gz: 3a2cdbec13b5e9a507fc348ed69ef3270e5fbd90275e03ba1797713e12d609c489eb1dd626592d4bd13aa66879c6bc2cd771343dc292a4862557e65e7f91eb22
+  metadata.gz: 4c8cd6915a54439599fc5a6f6373a6b72c2e678ecfa3ccf02d406a3342f2358b47bfe8b8dbe6661f74a59d622eda60fdd122bca2d5a9b8b95d0eb2d3d4cf6964
+  data.tar.gz: 16179171d941d670370d374cda08d683fa36e4553422b313b0c1e1db4c6649525c66ef6a466a7a23505cac98c9a086f074ec5ca153d233e34c906e322da51d00

data/build.gradle

@@ -19,7 +19,7 @@ configurations {
 dependencies {
     testImplementation "org.embulk:embulk-standards:0.9.15"
     testImplementation "org.embulk:embulk-test:0.9.15"
-    testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.2"
+    testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.4"
     testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
     testImplementation "org.junit.jupiter:junit-jupiter-engine:5.3.2"
 }

data/gradle.properties

@@ -1 +1 @@
-version=0.1.1
+version=0.2.0

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -9,7 +9,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
-com.github.kamatama41:embulk-test-helpers:0.7.2
+com.github.kamatama41:embulk-test-helpers:0.7.4
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0

data/src/main/java/org/embulk/output/multi/ControlDelegate.java

@@ -0,0 +1,40 @@
+package org.embulk.output.multi;
+
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.OutputPlugin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+class ControlDelegate implements OutputPlugin.Control {
+    private final int pluginIndex;
+    private final RunControlTask controlTask;
+
+    ControlDelegate(int index, RunControlTask controlTask) {
+        this.pluginIndex = index;
+        this.controlTask = controlTask;
+    }
+
+    @Override
+    public List<TaskReport> run(TaskSource taskSource) {
+        controlTask.addTaskSource(pluginIndex, taskSource);
+        List<TaskReport> reports;
+        try {
+            reports = controlTask.waitAndGetResult();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new RuntimeException(e.getCause());
+        }
+
+        final List<TaskReport> result = new ArrayList<>();
+        for (TaskReport taskReport : reports) {
+            final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+            result.add(report);
+        }
+        return result;
+    }
+}

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -1,7 +1,5 @@
 package org.embulk.output.multi;
 
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -18,31 +16,20 @@ import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CancellationException;
-import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 public class MultiOutputPlugin implements OutputPlugin {
     public interface PluginTask extends Task {
         @Config("outputs")
         List<ConfigSource> getOutputConfigs();
 
-        @Config("stop_on_failed_output")
-        @ConfigDefault("false")
-        boolean getStopOnFailedOutput();
-
         @Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
         @ConfigDefault("null")
         Optional<List<ConfigDiff>> getOutputConfigDiffs();
@@ -52,13 +39,7 @@ public class MultiOutputPlugin implements OutputPlugin {
     }
 
     private static final String CONFIG_NAME_OUTPUT_CONFIG_DIFFS = "output_config_diffs";
-    private static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
-    private static final Logger LOGGER = LoggerFactory.getLogger(MultiOutputPlugin.class);
-    private final ExecutorService executorService;
-
-    public MultiOutputPlugin() {
-        this.executorService = Executors.newCachedThreadPool();
-    }
+    static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
 
     @Override
     public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
@@ -67,8 +48,7 @@ public class MultiOutputPlugin implements OutputPlugin {
             throw new ConfigException("'outputs' must have more than than or equals to 1 element.");
         }
         final ExecSession session = Exec.session();
-        final RunControlTask controlTask = new RunControlTask(task, control, executorService);
-        controlTask.runAsynchronously();
+        final RunControlTask controlTask = new RunControlTask(task, control);
         return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
                 delegate.transaction(schema, taskCount, controlTask)
         ));
@@ -78,8 +58,7 @@ public class MultiOutputPlugin implements OutputPlugin {
     public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final ExecSession session = Exec.session();
-        final RunControlTask controlTask = new RunControlTask(task, control, executorService);
-        controlTask.runAsynchronously();
+        final RunControlTask controlTask = new RunControlTask(task, control);
         return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
                 delegate.resume(schema, taskCount, controlTask)
         ));
@@ -100,14 +79,14 @@ public class MultiOutputPlugin implements OutputPlugin {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final ExecSession session = Exec.session();
         final List<TransactionalPageOutputDelegate> delegates = mapWithPluginDelegate(task, session, delegate ->
-                new TransactionalPageOutputDelegate(delegate, delegate.open(schema, taskIndex), task.getStopOnFailedOutput())
+                new TransactionalPageOutputDelegate(taskIndex, delegate, delegate.open(schema, taskIndex))
         );
 
         return new TransactionalPageOutput() {
             @Override
             public void add(Page original) {
                 final Buffer originalBuffer = original.buffer();
-                for (TransactionalPageOutput output : delegates) {
+                for (TransactionalPageOutputDelegate output : delegates) {
                     final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
                     copiedBuffer.offset(originalBuffer.offset());
                     copiedBuffer.limit(originalBuffer.limit());
@@ -122,21 +101,21 @@ public class MultiOutputPlugin implements OutputPlugin {
 
             @Override
             public void finish() {
-                for (TransactionalPageOutput output : delegates) {
+                for (TransactionalPageOutputDelegate output : delegates) {
                     output.finish();
                 }
             }
 
             @Override
             public void close() {
-                for (TransactionalPageOutput output : delegates) {
+                for (TransactionalPageOutputDelegate output : delegates) {
                     output.close();
                 }
             }
 
             @Override
             public void abort() {
-                for (TransactionalPageOutput output : delegates) {
+                for (TransactionalPageOutputDelegate output : delegates) {
                     output.abort();
                 }
             }
@@ -145,8 +124,19 @@ public class MultiOutputPlugin implements OutputPlugin {
             public TaskReport commit() {
                 final TaskReport report = Exec.newTaskReport();
                 final List<TaskReport> reports = new ArrayList<>();
-                for (TransactionalPageOutput output : delegates) {
-                    reports.add(output.commit());
+                final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
+                for (TransactionalPageOutputDelegate output : delegates) {
+                    try {
+                        reports.add(output.commit());
+                    } catch (PluginExecutionException e) {
+                        errorPlugins.add(e.getPlugin());
+                    }
+                }
+                if (!errorPlugins.isEmpty()) {
+                    throw new RuntimeException(
+                            String.format("Following plugins failed to output [%s]",
+                                    errorPlugins.stream().map(OutputPluginDelegate::getPluginCode).collect(Collectors.joining(", "))
+                            ));
                 }
                 report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
                 return report;
@@ -154,239 +144,17 @@ public class MultiOutputPlugin implements OutputPlugin {
         };
     }
 
-    private static class RunControlTask implements Callable<List<TaskReport>> {
-        private final PluginTask task;
-        private final OutputPlugin.Control control;
-        private final ExecutorService executorService;
-        private final CountDownLatch latch;
-        private final TaskSource[] taskSources;
-        private Future<List<TaskReport>> future;
-
-        RunControlTask(PluginTask task, OutputPlugin.Control control, ExecutorService executorService) {
-            this.task = task;
-            this.control = control;
-            this.executorService = executorService;
-            this.latch = new CountDownLatch(task.getOutputConfigs().size());
-            this.taskSources = new TaskSource[task.getOutputConfigs().size()];
-        }
-
-        @Override
-        public List<TaskReport> call() throws Exception {
-            latch.await();
-            task.setTaskSources(Arrays.asList(taskSources));
-            return control.run(task.dump());
-        }
-
-        void runAsynchronously() {
-            future = executorService.submit(this);
-        }
-
-        void cancel() {
-            future.cancel(true);
-        }
-
-        void addTaskSource(int index, TaskSource taskSource) {
-            taskSources[index] = taskSource;
-            latch.countDown();
-        }
-
-        List<TaskReport> waitAndGetResult() throws ExecutionException, InterruptedException {
-            return future.get();
-        }
-    }
-
-    private static class OutputPluginDelegate {
-        private final int pluginIndex;
-        private final PluginType type;
-        private final OutputPlugin plugin;
-        private final ConfigSource config;
-        private final TaskSource taskSource;
-        private final ExecutorService executorService;
-
-        OutputPluginDelegate(
-                int pluginIndex,
-                PluginType type,
-                OutputPlugin plugin,
-                ConfigSource config,
-                TaskSource taskSource,
-                ExecutorService executorService
-        ) {
-            this.pluginIndex = pluginIndex;
-            this.type = type;
-            this.plugin = plugin;
-            this.config = config;
-            this.taskSource = taskSource;
-            this.executorService = executorService;
-        }
-
-        Future<ConfigDiff> transaction(Schema schema, int taskCount, RunControlTask controlTask) {
-            LOGGER.debug("Run transaction for {}", getPluginNameForLogging());
-            return executorService.submit(() -> {
-                try {
-                    return plugin.transaction(config, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
-                } catch (CancellationException e) {
-                    LOGGER.error("Canceled transaction for {} by other plugin's error", getPluginNameForLogging());
-                    throw e;
-                } catch (Exception e) {
-                    LOGGER.error("Transaction for {} failed.", getPluginNameForLogging(), e);
-                    controlTask.cancel();
-                    throw e;
-                }
-            });
-        }
-
-        Future<ConfigDiff> resume(Schema schema, int taskCount, RunControlTask controlTask) {
-            LOGGER.debug("Run resume for {}", getPluginNameForLogging());
-            return executorService.submit(() -> {
-                try {
-                    return plugin.resume(taskSource, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
-                } catch (CancellationException e) {
-                    LOGGER.error("Canceled resume for {} by other plugin's error", getPluginNameForLogging());
-                    throw e;
-                } catch (Exception e) {
-                    LOGGER.error("Resume for {} failed.", getPluginNameForLogging(), e);
-                    controlTask.cancel();
-                    throw e;
-                }
-            });
-        }
-
-        void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
-            LOGGER.debug("Run cleanup for {}", getPluginNameForLogging());
-            List<TaskReport> successReportsForPlugin = new ArrayList<>();
-            for (TaskReport successTaskReport : successTaskReports) {
-                final TaskReport report = successTaskReport.get(TaskReports.class, CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
-                successReportsForPlugin.add(report);
-                plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
-            }
-        }
-
-        TransactionalPageOutput open(Schema schema, int taskIndex) {
-            LOGGER.debug("Run open for {}", getPluginNameForLogging());
-            return plugin.open(taskSource, schema, taskIndex);
-        }
-
-        private String getPluginNameForLogging() {
-            return String.format("%s output plugin (pluginIndex: %s)", type.getName(), pluginIndex);
-        }
-    }
-
-    private static class TransactionalPageOutputDelegate implements TransactionalPageOutput {
-        private final OutputPluginDelegate source;
-        private final TransactionalPageOutput delegate;
-        private final boolean isStopOnFailedOutput;
-
-        TransactionalPageOutputDelegate(
-                OutputPluginDelegate source, TransactionalPageOutput delegate, boolean isStopOnFailedOutput) {
-            this.source = source;
-            this.delegate = delegate;
-            this.isStopOnFailedOutput = isStopOnFailedOutput;
-        }
-
-        @Override
-        public void add(Page page) {
-            try {
-                delegate.add(page);
-            } catch (Exception e) {
-                warnOrException(e);
-            }
-        }
-
-        @Override
-        public void finish() {
-            try {
-                delegate.finish();
-            } catch (Exception e) {
-                warnOrException(e);
-            }
-        }
-
-        @Override
-        public void close() {
-            try {
-                delegate.close();
-            } catch (Exception e) {
-                warnOrException(e);
-            }
-        }
-
-        @Override
-        public void abort() {
-            try {
-                delegate.abort();
-            } catch (Exception e) {
-                warnOrException(e);
-            }
-        }
-
-        @Override
-        public TaskReport commit() {
-            return delegate.commit();
-        }
-
-        private void warnOrException(Exception e) {
-            final String message = String.format("Failed on output for %s.", source.getPluginNameForLogging());
-            if (isStopOnFailedOutput) {
-                throw new RuntimeException(message, e);
-            }
-            LOGGER.warn(message);
-        }
-    }
-
-    private static class ControlDelegate implements OutputPlugin.Control {
-        private final int pluginIndex;
-        private final RunControlTask controlTask;
-
-        ControlDelegate(int index, RunControlTask controlTask) {
-            this.pluginIndex = index;
-            this.controlTask = controlTask;
-        }
-
-        @Override
-        public List<TaskReport> run(TaskSource taskSource) {
-            controlTask.addTaskSource(pluginIndex, taskSource);
-            List<TaskReport> reports;
-            try {
-                reports = controlTask.waitAndGetResult();
-            } catch (InterruptedException | ExecutionException e) {
-                throw new RuntimeException(e);
-            }
-
-            final List<TaskReport> result = new ArrayList<>();
-            for (TaskReport taskReport : reports) {
-                final TaskReport report = taskReport.get(TaskReports.class, CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
-                result.add(report);
-            }
-            return result;
-        }
-    }
-
-    private static class TaskReports {
-        private final List<TaskReport> reports;
-
-        @JsonCreator
-        TaskReports(@JsonProperty("reports") List<TaskReport> reports) {
-            this.reports = reports;
-        }
-
-        @JsonProperty("reports")
-        List<TaskReport> getReports() {
-            return reports;
-        }
-
-        TaskReport get(int index) {
-            return reports.get(index);
-        }
-    }
-
     private static ConfigDiff buildConfigDiff(List<Future<ConfigDiff>> runPluginTasks) {
         final ConfigDiff configDiff = Exec.newConfigDiff();
         List<ConfigDiff> configDiffs = new ArrayList<>();
         for (Future<ConfigDiff> pluginTask : runPluginTasks) {
             try {
                 configDiffs.add(pluginTask.get());
-            } catch (InterruptedException | ExecutionException e) {
+            } catch (InterruptedException e) {
+                Thread.currentThread().interrupt();
                 throw new RuntimeException(e);
+            } catch (ExecutionException e) {
+                throw new RuntimeException(e.getCause());
             }
         }
         configDiff.set(CONFIG_NAME_OUTPUT_CONFIG_DIFFS, configDiffs);
@@ -406,7 +174,7 @@ public class MultiOutputPlugin implements OutputPlugin {
             if (task.getTaskSources() != null) {
                 taskSource = task.getTaskSources().get(pluginIndex);
             }
-            result.add(action.apply(new OutputPluginDelegate(pluginIndex, pluginType, outputPlugin, config, taskSource, executorService)));
+            result.add(action.apply(new OutputPluginDelegate(pluginIndex, pluginType, outputPlugin, config, taskSource)));
         }
         return result;
     }

data/src/main/java/org/embulk/output/multi/OutputPluginDelegate.java

@@ -0,0 +1,111 @@
+package org.embulk.output.multi;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+class OutputPluginDelegate {
+    private static final Logger LOGGER = LoggerFactory.getLogger(OutputPluginDelegate.class);
+    private static final String THREAD_NAME_FORMAT = "multi-output-plugin-%s-%%d";
+    private final int pluginIndex;
+    private final PluginType type;
+    private final OutputPlugin plugin;
+    private final ConfigSource config;
+    private final TaskSource taskSource;
+    private final ExecutorService executorService;
+
+    OutputPluginDelegate(
+            int pluginIndex,
+            PluginType type,
+            OutputPlugin plugin,
+            ConfigSource config,
+            TaskSource taskSource
+    ) {
+        this.pluginIndex = pluginIndex;
+        this.type = type;
+        this.plugin = plugin;
+        this.config = config;
+        this.taskSource = taskSource;
+        this.executorService = Executors.newSingleThreadExecutor(
+                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, generatePluginCode(type, pluginIndex))).build()
+        );
+    }
+
+    Future<ConfigDiff> transaction(Schema schema, int taskCount, RunControlTask controlTask) {
+        return executorService.submit(() -> {
+            try {
+                LOGGER.debug("Run #transaction for {}", getPluginCode());
+                return plugin.transaction(config, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+            } catch (CancellationException e) {
+                LOGGER.error("Canceled #transaction for {} by other plugin's error", getPluginCode());
+                throw e;
+            } catch (Exception e) {
+                LOGGER.error("Transaction for {} failed.", getPluginCode(), e);
+                controlTask.cancel();
+                throw e;
+            } finally {
+                executorService.shutdown();
+            }
+        });
+    }
+
+    Future<ConfigDiff> resume(Schema schema, int taskCount, RunControlTask controlTask) {
+        return executorService.submit(() -> {
+            try {
+                LOGGER.debug("Run #resume for {}", getPluginCode());
+                return plugin.resume(taskSource, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+            } catch (CancellationException e) {
+                LOGGER.error("Canceled #resume for {} by other plugin's error", getPluginCode());
+                throw e;
+            } catch (Exception e) {
+                LOGGER.error("Resume for {} failed.", getPluginCode(), e);
+                controlTask.cancel();
+                throw e;
+            } finally {
+                executorService.shutdown();
+            }
+        });
+    }
+
+    void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
+        LOGGER.debug("Run #cleanup for {}", getPluginCode());
+        List<TaskReport> successReportsForPlugin = new ArrayList<>();
+        for (TaskReport successTaskReport : successTaskReports) {
+            final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+            successReportsForPlugin.add(report);
+        }
+        plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
+    }
+
+    TransactionalPageOutput open(Schema schema, int taskIndex) {
+        LOGGER.debug("Run #open for {}", getPluginCode());
+        return plugin.open(taskSource, schema, taskIndex);
+    }
+
+    PluginType getType() {
+        return type;
+    }
+
+    String getPluginCode() {
+        return generatePluginCode(type, pluginIndex);
+    }
+
+    private static String generatePluginCode(PluginType type, int pluginIndex) {
+        return String.format("%s-%d", type.getName(), pluginIndex);
+    }
+}

data/src/main/java/org/embulk/output/multi/PluginExecutionException.java

@@ -0,0 +1,15 @@
+package org.embulk.output.multi;
+
+class PluginExecutionException extends RuntimeException {
+
+    private final OutputPluginDelegate plugin;
+
+    PluginExecutionException(OutputPluginDelegate plugin, Throwable cause) {
+        super(cause);
+        this.plugin = plugin;
+    }
+
+    OutputPluginDelegate getPlugin() {
+        return plugin;
+    }
+}

data/src/main/java/org/embulk/output/multi/RunControlTask.java

@@ -0,0 +1,65 @@
+package org.embulk.output.multi;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.OutputPlugin;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicReferenceArray;
+
+class RunControlTask implements Callable<List<TaskReport>> {
+    private static final String THREAD_NAME_FORMAT = "multi-run-control-%d";
+    private final MultiOutputPlugin.PluginTask task;
+    private final OutputPlugin.Control control;
+    private final CountDownLatch latch;
+    private final AtomicReferenceArray<TaskSource> taskSources;
+    private final ExecutorService executorService;
+    private Future<List<TaskReport>> result;
+
+    RunControlTask(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
+        this.task = task;
+        this.control = control;
+        this.latch = new CountDownLatch(task.getOutputConfigs().size());
+        this.taskSources = new AtomicReferenceArray<>(task.getOutputConfigs().size());
+        this.executorService = Executors.newSingleThreadExecutor(
+                new ThreadFactoryBuilder().setNameFormat(THREAD_NAME_FORMAT).build()
+        );
+        this.result = executorService.submit(this);
+    }
+
+    @Override
+    public List<TaskReport> call() throws Exception {
+        latch.await();
+        List<TaskSource> taskSources = new ArrayList<>(this.taskSources.length());
+        for (int i = 0; i < this.taskSources.length(); i++) {
+            taskSources.add(this.taskSources.get(i));
+        }
+        task.setTaskSources(taskSources);
+        return control.run(task.dump());
+    }
+
+    void cancel() {
+        result.cancel(true);
+    }
+
+    void addTaskSource(int index, TaskSource taskSource) {
+        taskSources.set(index, taskSource);
+        latch.countDown();
+    }
+
+    List<TaskReport> waitAndGetResult() throws ExecutionException, InterruptedException {
+        try {
+            return result.get();
+        } finally {
+            executorService.shutdown();
+        }
+    }
+}

data/src/main/java/org/embulk/output/multi/TaskReports.java

@@ -0,0 +1,25 @@
+package org.embulk.output.multi;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.config.TaskReport;
+
+import java.util.List;
+
+class TaskReports {
+    private final List<TaskReport> reports;
+
+    @JsonCreator
+    TaskReports(@JsonProperty("reports") List<TaskReport> reports) {
+        this.reports = reports;
+    }
+
+    @JsonProperty("reports")
+    List<TaskReport> getReports() {
+        return reports;
+    }
+
+    TaskReport get(int index) {
+        return reports.get(index);
+    }
+}

data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java

@@ -0,0 +1,93 @@
+package org.embulk.output.multi;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Page;
+import org.embulk.spi.TransactionalPageOutput;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.function.Supplier;
+
+class TransactionalPageOutputDelegate {
+    private static final String THREAD_NAME_FORMAT = "multi-page-output-%s-%d";
+    private final OutputPluginDelegate source;
+    private final TransactionalPageOutput delegate;
+    private final BlockingQueue<Supplier<Object>> taskQueue;
+    private final ExecutorService executorService;
+    private final Future<TaskReport> result;
+
+    TransactionalPageOutputDelegate(
+            int taskIndex,
+            OutputPluginDelegate source,
+            TransactionalPageOutput delegate
+    ) {
+        this.source = source;
+        this.delegate = delegate;
+        this.taskQueue = new LinkedBlockingQueue<>();
+        this.executorService = Executors.newSingleThreadExecutor(
+                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getPluginCode(), taskIndex)).build()
+        );
+        this.result = executorService.submit(new Worker());
+    }
+
+    void add(Page page) {
+        taskQueue.add(() -> {
+            delegate.add(page);
+            return null;
+        });
+    }
+
+    void finish() {
+        taskQueue.add(() -> {
+            delegate.finish();
+            return null;
+        });
+    }
+
+    void close() {
+        taskQueue.add(() -> {
+            delegate.close();
+            return null;
+        });
+    }
+
+    void abort() {
+        taskQueue.add(() -> {
+            delegate.abort();
+            return null;
+        });
+    }
+
+    TaskReport commit() {
+        taskQueue.add(delegate::commit);
+        try {
+            return result.get();
+        } catch (InterruptedException e) {
+            Thread.currentThread().interrupt();
+            throw new RuntimeException(e);
+        } catch (ExecutionException e) {
+            throw new PluginExecutionException(source, e.getCause());
+        } finally {
+            executorService.shutdown();
+        }
+    }
+
+    private class Worker implements Callable<TaskReport> {
+        @Override
+        public TaskReport call() throws InterruptedException {
+            while (true) {
+                final Supplier<Object> task = taskQueue.take();
+                final Object result = task.get();
+                if (result instanceof TaskReport) {
+                    return (TaskReport) result;
+                }
+            }
+        }
+    }
+}

data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

@@ -25,7 +25,6 @@ import static org.embulk.test.Utils.record;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
-import static org.junit.jupiter.api.Assertions.fail;
 
 @EmbulkTest(MultiOutputPlugin.class)
 class TestMultiOutputPlugin extends EmbulkPluginTest {
@@ -116,14 +115,16 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
     }
 
     @Test
-    void testStopOnFailedOutputIsFalse() throws IOException {
+    void testAnOutputFailedFailedAfterOpen() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
-        final ConfigSource outConfig = configFromResource("yaml/out_stop_on_failed_output.yml");
+        final ConfigSource outConfig = configFromResource("yaml/out_failed_output_after_open.yml");
         final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
         System.setOut(new PrintStream(outputStream));
 
         // Run Embulk without error
-        runOutput(inConfig, outConfig);
+        try {
+            runOutput(inConfig, outConfig);
+        } catch (PartialExecutionException ignored) {}
 
         // All records should be loaded for local_object output plugin.
         final Record[] values = new Record[]{record(1, "user1", 20), record(2, "user2", 21)};
@@ -137,20 +138,16 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
     }
 
     @Test
-    void testStopOnFailedOutputIsTrue() throws IOException {
+    void testAnOutputFailedFailedBeforeOpen() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
-        final ConfigSource outConfig = configFromResource("yaml/out_stop_on_failed_output.yml")
-                .set("stop_on_failed_output", true);
+        final ConfigSource outConfig = configFromResource("yaml/out_failed_output_before_open.yml");
         final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
         System.setOut(new PrintStream(outputStream));
 
-        // Run Embulk
+        // Run Embulk without error
         try {
             runOutput(inConfig, outConfig);
-            fail("No exception");
-        } catch (PartialExecutionException e) {
-            assertTrue(e.getMessage().contains("Failed on output for throw_exception output plugin (pluginIndex: 0)"));
-        }
+        } catch (PartialExecutionException ignored) {}
 
         // No record should be loaded for local_object output plugin.
         assertRecords();

data/src/test/resources/yaml/out_failed_output_before_open.yml

@@ -0,0 +1,13 @@
+type: multi
+outputs:
+  - type: throw_exception
+    thrown_on: COMMIT
+    source:
+      type: stdout
+  - type: local_object
+    incremental: true
+    incremental_column: id
+  - type: throw_exception
+    thrown_on: BEFORE_TRANSACTION
+    source:
+      type: stdout

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-multi
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Shinichi ISHIMURA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-03 00:00:00.000000000 Z
+date: 2019-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-output-multi-0.1.1.jar
+- classpath/embulk-output-multi-0.2.0.jar
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile
@@ -60,11 +60,18 @@ files:
 - gradlew.bat
 - lib/embulk/output/multi.rb
 - settings.gradle
+- src/main/java/org/embulk/output/multi/ControlDelegate.java
 - src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
+- src/main/java/org/embulk/output/multi/OutputPluginDelegate.java
+- src/main/java/org/embulk/output/multi/PluginExecutionException.java
+- src/main/java/org/embulk/output/multi/RunControlTask.java
+- src/main/java/org/embulk/output/multi/TaskReports.java
+- src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java
 - src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
 - src/test/resources/yaml/in_base.yml
 - src/test/resources/yaml/out_base.yml
-- src/test/resources/yaml/out_stop_on_failed_output.yml
+- src/test/resources/yaml/out_failed_output_after_open.yml
+- src/test/resources/yaml/out_failed_output_before_open.yml
 homepage: https://github.com/kamatama41/embulk-output-multi
 licenses:
 - MIT