embulk-output-multi 0.2.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40d2c979fbce0853ed920cae18f0f867284e9491
-  data.tar.gz: e02bef302e6b8061f9029dc4d7077a00b7563623
+  metadata.gz: d23fe45b21d16664727849db0d1af9015a180834
+  data.tar.gz: 5b3dbb7047049be9dd3784cafb93855e1288fc36
 SHA512:
-  metadata.gz: ba665f2f4108d65c1cf48ad5a94789e6d719443bb0e8bc7f579f05357407918ab275b41cd55aa96af0ae0854e572427f55039a8502e332c8ab22d6707d6118f9
-  data.tar.gz: 7fa76fd0c1a8c06e89b2b0eff90e6b16e00985db269444a37812da6af0f40e3a74a748b00c7e3dfc675b35f7c60875e5f57efd87bf6963eacf57f05cc04a9e79
+  metadata.gz: 0fe2a6a708a3602be1e8512eba3d2ab3d9e4fd789ce833a0f5872f7c9790adc4112463d0ed5f84e45224bff331ef2594c4cff9596288c91be8144e04ce7ad7a9
+  data.tar.gz: 6b66ca734eb9786edd92632ede8678c2152a22b4ffa668215ea12d3ebc9250364582c0399e30ab130ce7ab2ff8de4dd245b34b8b61e2c10ef30ed34656ece096

data/README.md

@@ -1,11 +1,11 @@
 # Multi output plugin for Embulk
 
-This plugin can copies an output to multiple destinations.
+This plugin copies an output to multiple destinations.
 
 ### Notes
 - It's still very experimental version, so might change its configuration names or styles without notification. 
-- It might have performance issues or bugs when loading large records.
-- It might not working on other executors than LocalExecutor.
+- As this plugin performs multiple output methods, it might have a performance issue with large records.
+- It might not work on other executors than LocalExecutor.
 - I would appreciate it if you use this and give me reports/feedback!
 
 ## Overview
@@ -27,13 +27,13 @@ out:
   outputs:
     # Output to stdout
     - type: stdout
-    # Output to file as CSV
+    # Output to files as CSV
     - type: file
       path_prefix: out_file_
       file_ext: csv
       formatter:
         type: csv
-    # Output to file as TSV
+    # Output to files as TSV
     - type: file
       path_prefix: out_file_
       file_ext: tsv

data/build.gradle

@@ -1,5 +1,5 @@
 plugins {
-    id "com.github.kamatama41.embulk" version "0.4.0"
+    id "com.github.kamatama41.embulk" version "0.5.2"
     id "net.researchgate.release" version "2.8.0"
 }
 
@@ -17,8 +17,10 @@ configurations {
 }
 
 dependencies {
-    testImplementation "org.embulk:embulk-standards:0.9.15"
-    testImplementation "org.embulk:embulk-test:0.9.15"
+    testImplementation "org.embulk:embulk-deps-buffer:0.9.23"
+    testImplementation "org.embulk:embulk-deps-config:0.9.23"
+    testImplementation "org.embulk:embulk-standards:0.9.23"
+    testImplementation "org.embulk:embulk-test:0.9.23"
     testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.4"
     testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
     testImplementation "org.junit.jupiter:junit-jupiter-engine:5.3.2"
@@ -29,7 +31,7 @@ test {
 }
 
 embulk {
-    version = "0.9.15"
+    version = "0.9.23"
     category = "output"
     name = "multi"
     authors = ["Shinichi ISHIMURA"]

data/gradle.properties

@@ -1 +1 @@
-version=0.2.1
+version=0.5.0

data/gradle/dependency-locks/compileClasspath.lockfile

@@ -9,27 +9,17 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
-com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0
 com.google.inject:guice:4.0
-com.ibm.icu:icu4j:54.1.1
 commons-beanutils:commons-beanutils-core:1.8.3
-commons-cli:commons-cli:1.3.1
-commons-collections:commons-collections:3.2.1
-commons-lang:commons-lang:2.4
-io.airlift:slice:0.9
-io.netty:netty-buffer:4.0.44.Final
-io.netty:netty-common:4.0.44.Final
 javax.inject:javax.inject:1
 javax.validation:validation-api:1.1.0.Final
 joda-time:joda-time:2.9.2
 org.apache.bval:bval-core:0.5
 org.apache.bval:bval-jsr303:0.5
 org.apache.commons:commons-lang3:3.4
-org.apache.velocity:velocity:1.7
-org.embulk:embulk-core:0.9.15
+org.embulk:embulk-core:0.9.23
 org.jruby:jruby-complete:9.1.15.0
 org.msgpack:msgpack-core:0.8.11
 org.slf4j:slf4j-api:1.7.12
-org.yaml:snakeyaml:1.18

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -10,15 +10,10 @@ com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
 com.github.kamatama41:embulk-test-helpers:0.7.4
-com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0
 com.google.inject:guice:4.0
-com.ibm.icu:icu4j:54.1.1
 commons-beanutils:commons-beanutils-core:1.8.3
-commons-cli:commons-cli:1.3.1
-commons-collections:commons-collections:3.2.1
-commons-lang:commons-lang:2.4
 io.airlift:slice:0.9
 io.netty:netty-buffer:4.0.44.Final
 io.netty:netty-common:4.0.44.Final
@@ -30,11 +25,12 @@ org.apache.bval:bval-core:0.5
 org.apache.bval:bval-jsr303:0.5
 org.apache.commons:commons-compress:1.10
 org.apache.commons:commons-lang3:3.4
-org.apache.velocity:velocity:1.7
 org.apiguardian:apiguardian-api:1.0.0
-org.embulk:embulk-core:0.9.15
-org.embulk:embulk-standards:0.9.15
-org.embulk:embulk-test:0.9.15
+org.embulk:embulk-core:0.9.23
+org.embulk:embulk-deps-buffer:0.9.23
+org.embulk:embulk-deps-config:0.9.23
+org.embulk:embulk-standards:0.9.23
+org.embulk:embulk-test:0.9.23
 org.hamcrest:hamcrest-core:1.3
 org.hamcrest:hamcrest-library:1.3
 org.jruby:jruby-complete:9.1.15.0

data/src/main/java/org/embulk/output/multi/AsyncRunControl.java

@@ -5,45 +5,46 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.OutputPlugin;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 class AsyncRunControl {
     private static final String THREAD_NAME_FORMAT = "multi-run-control-%d";
     private final MultiOutputPlugin.PluginTask task;
     private final OutputPlugin.Control control;
     private final CountDownLatch latch;
-    private final AtomicReferenceArray<TaskSource> taskSources;
+    private final ConcurrentMap<String, TaskSource> taskSources;
     private final ExecutorService executorService;
-    private Future<List<TaskReport>> result;
+    private final Future<List<TaskReport>> result;
 
     static AsyncRunControl start(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
-        return new AsyncRunControl(task, control).start();
+        return new AsyncRunControl(task, control);
     }
 
     private AsyncRunControl(MultiOutputPlugin.PluginTask task, OutputPlugin.Control control) {
         this.task = task;
         this.control = control;
         this.latch = new CountDownLatch(task.getOutputConfigs().size());
-        this.taskSources = new AtomicReferenceArray<>(task.getOutputConfigs().size());
+        this.taskSources = new ConcurrentHashMap<>(task.getOutputConfigs().size());
         this.executorService = Executors.newSingleThreadExecutor(
                 new ThreadFactoryBuilder().setNameFormat(THREAD_NAME_FORMAT).build()
         );
+        this.result = executorService.submit(new RunControl());
     }
 
     void cancel() {
         result.cancel(true);
     }
 
-    void addTaskSource(int index, TaskSource taskSource) {
-        taskSources.set(index, taskSource);
+    void addTaskSource(String tag, TaskSource taskSource) {
+        taskSources.putIfAbsent(tag, taskSource);
         latch.countDown();
     }
 
@@ -55,19 +56,10 @@ class AsyncRunControl {
         }
     }
 
-    private AsyncRunControl start() {
-        this.result = executorService.submit(new RunControl());
-        return this;
-    }
-
     private class RunControl implements Callable<List<TaskReport>> {
         @Override
         public List<TaskReport> call() throws Exception {
             latch.await();
-            List<TaskSource> taskSources = new ArrayList<>(AsyncRunControl.this.taskSources.length());
-            for (int i = 0; i < AsyncRunControl.this.taskSources.length(); i++) {
-                taskSources.add(AsyncRunControl.this.taskSources.get(i));
-            }
             task.setTaskSources(taskSources);
             return control.run(task.dump());
         }

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -9,21 +9,21 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.plugin.PluginType;
-import org.embulk.spi.Buffer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.Page;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.function.Function;
-import java.util.stream.Collectors;
 
 public class MultiOutputPlugin implements OutputPlugin {
     public interface PluginTask extends Task {
@@ -32,12 +32,13 @@ public class MultiOutputPlugin implements OutputPlugin {
 
         @Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
         @ConfigDefault("null")
-        Optional<List<ConfigDiff>> getOutputConfigDiffs();
+        Optional<Map<String, ConfigDiff>> getOutputConfigDiffs();
 
-        List<TaskSource> getTaskSources();
-        void setTaskSources(List<TaskSource> taskSources);
+        Map<String, TaskSource> getTaskSources();
+        void setTaskSources(Map<String, TaskSource> taskSources);
     }
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(MultiOutputPlugin.class);
     private static final String CONFIG_NAME_OUTPUT_CONFIG_DIFFS = "output_config_diffs";
     static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
 
@@ -45,7 +46,7 @@ public class MultiOutputPlugin implements OutputPlugin {
     public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
         final PluginTask task = config.loadConfig(PluginTask.class);
         if (task.getOutputConfigs().isEmpty()) {
-            throw new ConfigException("'outputs' must have more than than or equals to 1 element.");
+            throw new ConfigException("'outputs' must have more than or equals to 1 element.");
         }
         final ExecSession session = Exec.session();
         final AsyncRunControl runControl = AsyncRunControl.start(task, control);
@@ -78,78 +79,15 @@ public class MultiOutputPlugin implements OutputPlugin {
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final ExecSession session = Exec.session();
-        final List<TransactionalPageOutputDelegate> delegates = mapWithPluginDelegate(task, session, delegate ->
-                new TransactionalPageOutputDelegate(taskIndex, delegate, delegate.open(schema, taskIndex))
-        );
-
-        return new TransactionalPageOutput() {
-            @Override
-            public void add(Page original) {
-                final Buffer originalBuffer = original.buffer();
-                for (TransactionalPageOutputDelegate output : delegates) {
-                    final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
-                    copiedBuffer.offset(originalBuffer.offset());
-                    copiedBuffer.limit(originalBuffer.limit());
-
-                    final Page copiedPage = Page.wrap(copiedBuffer);
-                    copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
-                    copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
-
-                    output.add(copiedPage);
-                }
-            }
-
-            @Override
-            public void finish() {
-                for (TransactionalPageOutputDelegate output : delegates) {
-                    output.finish();
-                }
-            }
-
-            @Override
-            public void close() {
-                for (TransactionalPageOutputDelegate output : delegates) {
-                    output.close();
-                }
-            }
-
-            @Override
-            public void abort() {
-                for (TransactionalPageOutputDelegate output : delegates) {
-                    output.abort();
-                }
-            }
-
-            @Override
-            public TaskReport commit() {
-                final TaskReport report = Exec.newTaskReport();
-                final List<TaskReport> reports = new ArrayList<>();
-                final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
-                for (TransactionalPageOutputDelegate output : delegates) {
-                    try {
-                        reports.add(output.commit());
-                    } catch (PluginExecutionException e) {
-                        errorPlugins.add(e.getPlugin());
-                    }
-                }
-                if (!errorPlugins.isEmpty()) {
-                    throw new RuntimeException(
-                            String.format("Following plugins failed to output [%s]",
-                                    errorPlugins.stream().map(OutputPluginDelegate::getPluginCode).collect(Collectors.joining(", "))
-                            ));
-                }
-                report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
-                return report;
-            }
-        };
+        return MultiTransactionalPageOutput.open(schema, taskIndex, mapWithPluginDelegate(task, session, Function.identity()));
     }
 
-    private static ConfigDiff buildConfigDiff(List<Future<ConfigDiff>> runPluginTasks) {
+    private static ConfigDiff buildConfigDiff(List<OutputPluginDelegate.Transaction> transactions) {
         final ConfigDiff configDiff = Exec.newConfigDiff();
-        List<ConfigDiff> configDiffs = new ArrayList<>();
-        for (Future<ConfigDiff> pluginTask : runPluginTasks) {
+        Map<String, ConfigDiff> configDiffs = new HashMap<>();
+        for (OutputPluginDelegate.Transaction transaction: transactions) {
             try {
-                configDiffs.add(pluginTask.get());
+                configDiffs.put(transaction.getTag(), transaction.getResult());
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 throw new RuntimeException(e);
@@ -163,18 +101,29 @@ public class MultiOutputPlugin implements OutputPlugin {
 
     private <T> List<T> mapWithPluginDelegate(PluginTask task, ExecSession session, Function<OutputPluginDelegate, T> action) {
         List<T> result = new ArrayList<>();
-        for (int pluginIndex = 0; pluginIndex < task.getOutputConfigs().size(); pluginIndex++) {
-            final ConfigSource config = task.getOutputConfigs().get(pluginIndex);
-            if (task.getOutputConfigDiffs().isPresent()) {
-                config.merge(task.getOutputConfigDiffs().get().get(pluginIndex));
-            }
+        for (int i = 0; i < task.getOutputConfigs().size(); i++) {
+            final ConfigSource config = task.getOutputConfigs().get(i);
             final PluginType pluginType = config.get(PluginType.class, "type");
             final OutputPlugin outputPlugin = session.newPlugin(OutputPlugin.class, pluginType);
+
+            final String tag = String.format("%s_%d", pluginType.getName(), i);
+
+            // Merge ConfigDiff if exists
+            if (task.getOutputConfigDiffs().isPresent()) {
+                final ConfigDiff configDiff = task.getOutputConfigDiffs().get().get(tag);
+                if (configDiff != null) {
+                    config.merge(configDiff);
+                } else {
+                    LOGGER.debug("ConfigDiff for '{}' not found.", tag);
+                }
+            }
+            // Set TaskSource if exists
             TaskSource taskSource = null;
             if (task.getTaskSources() != null) {
-                taskSource = task.getTaskSources().get(pluginIndex);
+                taskSource = task.getTaskSources().get(tag);
             }
-            result.add(action.apply(new OutputPluginDelegate(pluginIndex, pluginType, outputPlugin, config, taskSource)));
+
+            result.add(action.apply(new OutputPluginDelegate(tag, outputPlugin, config, taskSource)));
         }
         return result;
     }

data/src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java

@@ -0,0 +1,137 @@
+package org.embulk.output.multi;
+
+import org.embulk.config.TaskReport;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Page;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+public class MultiTransactionalPageOutput implements TransactionalPageOutput {
+    private static final Logger LOGGER = LoggerFactory.getLogger(MultiTransactionalPageOutput.class);
+    private final int taskIndex;
+    private final List<Delegate> delegates;
+
+    static MultiTransactionalPageOutput open(Schema schema, int taskIndex, List<OutputPluginDelegate> plugins) {
+        return new MultiTransactionalPageOutput(
+                taskIndex,
+                plugins.stream()
+                        .map(plugin -> new Delegate(plugin, plugin.open(schema, taskIndex)))
+                        .collect(Collectors.toList())
+        );
+    }
+
+    private MultiTransactionalPageOutput(int taskIndex, List<Delegate> delegates) {
+        this.taskIndex = taskIndex;
+        this.delegates = delegates;
+    }
+
+    @Override
+    public void add(Page page) {
+        applyToAllPlugins(delegate -> delegate.add(copyPage(page)));
+        page.release();
+    }
+
+    @Override
+    public void finish() {
+        applyToAllPlugins(Delegate::finish);
+    }
+
+    @Override
+    public void close() {
+        applyToAllPlugins(Delegate::close);
+    }
+
+    @Override
+    public void abort() {
+        applyToAllPlugins(Delegate::abort);
+    }
+
+    @Override
+    public TaskReport commit() {
+        final Map<String, TaskReport> reports = new HashMap<>();
+        applyToAllPlugins(delegate -> reports.put(delegate.getTag(), delegate.commit()));
+        final TaskReport report = Exec.newTaskReport();
+        report.set(MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
+        return report;
+    }
+
+    private void applyToAllPlugins(Consumer<Delegate> command) {
+        final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
+        for (Delegate delegate : delegates) {
+            try {
+                command.accept(delegate);
+            } catch (Exception e) {
+                LOGGER.warn(String.format("Output for %s on index %d failed.", delegate.plugin.getTag(), taskIndex), e);
+                errorPlugins.add(delegate.plugin);
+            }
+        }
+
+        if (!errorPlugins.isEmpty()) {
+            throw new RuntimeException(
+                    String.format("Following plugins failed to output [%s] on index %d",
+                            errorPlugins.stream().map(OutputPluginDelegate::getTag).collect(Collectors.joining(", ")),
+                            taskIndex
+                    ));
+        }
+    }
+
+    private static Page copyPage(Page original) {
+        final Buffer originalBuffer = original.buffer();
+        final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array(), originalBuffer.offset(), originalBuffer.capacity());
+        copiedBuffer.limit(originalBuffer.limit());
+
+        final Page copiedPage = Page.wrap(copiedBuffer);
+        copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
+        copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
+        return copiedPage;
+    }
+
+    static class Delegate implements TransactionalPageOutput {
+        private final OutputPluginDelegate plugin;
+        private final TransactionalPageOutput output;
+
+        private Delegate(OutputPluginDelegate plugin, TransactionalPageOutput output) {
+            this.plugin = plugin;
+            this.output = output;
+        }
+
+        @Override
+        public void add(Page page) {
+            output.add(page);
+        }
+
+        @Override
+        public void finish() {
+            output.finish();
+        }
+
+        @Override
+        public void close() {
+            output.close();
+        }
+
+        @Override
+        public void abort() {
+            output.abort();
+        }
+
+        @Override
+        public TaskReport commit() {
+            return output.commit();
+        }
+
+        String getTag() {
+            return plugin.getTag();
+        }
+    }
+}

data/src/main/java/org/embulk/output/multi/OutputPluginDelegate.java

@@ -5,7 +5,6 @@ import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.plugin.PluginType;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
@@ -23,105 +22,92 @@ import java.util.concurrent.Future;
 class OutputPluginDelegate {
     private static final Logger LOGGER = LoggerFactory.getLogger(OutputPluginDelegate.class);
     private static final String THREAD_NAME_FORMAT = "multi-output-plugin-%s-%%d";
-    private final int pluginIndex;
-    private final PluginType type;
+    private final String tag;
     private final OutputPlugin plugin;
     private final ConfigSource config;
     private final TaskSource taskSource;
     private final ExecutorService executorService;
 
     OutputPluginDelegate(
-            int pluginIndex,
-            PluginType type,
+            String tag,
             OutputPlugin plugin,
             ConfigSource config,
             TaskSource taskSource
     ) {
-        this.pluginIndex = pluginIndex;
-        this.type = type;
+        this.tag = tag;
         this.plugin = plugin;
         this.config = config;
         this.taskSource = taskSource;
         this.executorService = Executors.newSingleThreadExecutor(
-                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, generatePluginCode(type, pluginIndex))).build()
+                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, tag)).build()
         );
     }
 
-    Future<ConfigDiff> transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
-        return executorService.submit(() -> {
+    Transaction transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
+        return new Transaction(executorService.submit(() -> {
             try {
-                LOGGER.debug("Run #transaction for {}", getPluginCode());
-                return plugin.transaction(config, schema, taskCount, new Control(pluginIndex, runControl));
+                LOGGER.debug("Run #transaction for {}", getTag());
+                return plugin.transaction(config, schema, taskCount, new Control(runControl));
             } catch (CancellationException e) {
-                LOGGER.error("Canceled #transaction for {} by other plugin's error", getPluginCode());
+                LOGGER.error("Canceled #transaction for {} by other plugin's error", getTag());
                 throw e;
             } catch (Exception e) {
-                LOGGER.error("Transaction for {} failed.", getPluginCode(), e);
+                LOGGER.error("Transaction for {} failed.", getTag(), e);
                 runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
             }
-        });
+        }));
     }
 
-    Future<ConfigDiff> resume(Schema schema, int taskCount, AsyncRunControl runControl) {
-        return executorService.submit(() -> {
+    Transaction resume(Schema schema, int taskCount, AsyncRunControl runControl) {
+        return new Transaction(executorService.submit(() -> {
             try {
-                LOGGER.debug("Run #resume for {}", getPluginCode());
-                return plugin.resume(taskSource, schema, taskCount, new Control(pluginIndex, runControl));
+                LOGGER.debug("Run #resume for {}", getTag());
+                return plugin.resume(taskSource, schema, taskCount, new Control(runControl));
             } catch (CancellationException e) {
-                LOGGER.error("Canceled #resume for {} by other plugin's error", getPluginCode());
+                LOGGER.error("Canceled #resume for {} by other plugin's error", getTag());
                 throw e;
             } catch (Exception e) {
-                LOGGER.error("Resume for {} failed.", getPluginCode(), e);
+                LOGGER.error("Resume for {} failed.", getTag(), e);
                 runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
             }
-        });
+        }));
     }
 
     void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
-        LOGGER.debug("Run #cleanup for {}", getPluginCode());
+        LOGGER.debug("Run #cleanup for {}", getTag());
         List<TaskReport> successReportsForPlugin = new ArrayList<>();
         for (TaskReport successTaskReport : successTaskReports) {
-            final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+            final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
             successReportsForPlugin.add(report);
         }
         plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
     }
 
     TransactionalPageOutput open(Schema schema, int taskIndex) {
-        LOGGER.debug("Run #open for {}", getPluginCode());
+        LOGGER.debug("Run #open for {}", getTag());
         return plugin.open(taskSource, schema, taskIndex);
     }
 
-    PluginType getType() {
-        return type;
+    String getTag() {
+        return tag;
     }
 
-    String getPluginCode() {
-        return generatePluginCode(type, pluginIndex);
-    }
-
-    private static String generatePluginCode(PluginType type, int pluginIndex) {
-        return String.format("%s-%d", type.getName(), pluginIndex);
-    }
-
-    private static class Control implements OutputPlugin.Control {
-        private final int pluginIndex;
+    private class Control implements OutputPlugin.Control {
         private final AsyncRunControl runControl;
 
-        Control(int index, AsyncRunControl runControl) {
-            this.pluginIndex = index;
+        Control(AsyncRunControl runControl) {
             this.runControl = runControl;
         }
 
         @Override
         public List<TaskReport> run(TaskSource taskSource) {
-            runControl.addTaskSource(pluginIndex, taskSource);
+            runControl.addTaskSource(tag, taskSource);
             List<TaskReport> reports;
             try {
                 reports = runControl.waitAndGetResult();
@@ -134,10 +120,26 @@ class OutputPluginDelegate {
 
             final List<TaskReport> result = new ArrayList<>();
             for (TaskReport taskReport : reports) {
-                final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+                final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
                 result.add(report);
             }
             return result;
         }
     }
+
+    class Transaction {
+        private final Future<ConfigDiff> future;
+
+        private Transaction(Future<ConfigDiff> future) {
+            this.future = future;
+        }
+
+        String getTag() {
+            return OutputPluginDelegate.this.getTag();
+        }
+
+        ConfigDiff getResult() throws ExecutionException, InterruptedException {
+            return future.get();
+        }
+    }
 }

data/src/main/java/org/embulk/output/multi/TaskReports.java

@@ -4,22 +4,22 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.config.TaskReport;
 
-import java.util.List;
+import java.util.Map;
 
 class TaskReports {
-    private final List<TaskReport> reports;
+    private final Map<String, TaskReport> reports;
 
     @JsonCreator
-    TaskReports(@JsonProperty("reports") List<TaskReport> reports) {
+    TaskReports(@JsonProperty("reports") Map<String, TaskReport> reports) {
         this.reports = reports;
     }
 
     @JsonProperty("reports")
-    List<TaskReport> getReports() {
+    Map<String, TaskReport> getReports() {
         return reports;
     }
 
-    TaskReport get(int index) {
-        return reports.get(index);
+    TaskReport get(String tag) {
+        return reports.get(tag);
     }
 }

data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

@@ -24,10 +24,10 @@ import static org.embulk.test.Utils.configFromResource;
 import static org.embulk.test.Utils.record;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @EmbulkTest(MultiOutputPlugin.class)
 class TestMultiOutputPlugin extends EmbulkPluginTest {
+
     @Test
     void testMultipleOutputWorking() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
@@ -115,7 +115,7 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
     }
 
     @Test
-    void testAnOutputFailedFailedAfterOpen() throws IOException {
+    void testAnOutputFailedAfterOpen() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
         final ConfigSource outConfig = configFromResource("yaml/out_failed_output_after_open.yml");
         final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
@@ -138,7 +138,7 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
     }
 
     @Test
-    void testAnOutputFailedFailedBeforeOpen() throws IOException {
+    void testAnOutputFailedBeforeOpen() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
         final ConfigSource outConfig = configFromResource("yaml/out_failed_output_before_open.yml");
         final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-multi
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Shinichi ISHIMURA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-07 00:00:00.000000000 Z
+date: 2021-01-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-output-multi-0.2.1.jar
+- classpath/embulk-output-multi-0.5.0.jar
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile
@@ -62,10 +62,9 @@ files:
 - settings.gradle
 - src/main/java/org/embulk/output/multi/AsyncRunControl.java
 - src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
+- src/main/java/org/embulk/output/multi/MultiTransactionalPageOutput.java
 - src/main/java/org/embulk/output/multi/OutputPluginDelegate.java
-- src/main/java/org/embulk/output/multi/PluginExecutionException.java
 - src/main/java/org/embulk/output/multi/TaskReports.java
-- src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java
 - src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
 - src/test/resources/yaml/in_base.yml
 - src/test/resources/yaml/out_base.yml

data/src/main/java/org/embulk/output/multi/PluginExecutionException.java

@@ -1,15 +0,0 @@
-package org.embulk.output.multi;
-
-class PluginExecutionException extends RuntimeException {
-
-    private final OutputPluginDelegate plugin;
-
-    PluginExecutionException(OutputPluginDelegate plugin, Throwable cause) {
-        super(cause);
-        this.plugin = plugin;
-    }
-
-    OutputPluginDelegate getPlugin() {
-        return plugin;
-    }
-}

data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java

@@ -1,92 +0,0 @@
-package org.embulk.output.multi;
-
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.Page;
-import org.embulk.spi.TransactionalPageOutput;
-
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.function.Supplier;
-
-class TransactionalPageOutputDelegate {
-    private static final String THREAD_NAME_FORMAT = "multi-page-output-%s-%d";
-    private final OutputPluginDelegate source;
-    private final TransactionalPageOutput delegate;
-    private final BlockingQueue<Supplier<Object>> taskQueue;
-    private final ExecutorService executorService;
-    private final Future<TaskReport> result;
-
-    TransactionalPageOutputDelegate(
-            int taskIndex,
-            OutputPluginDelegate source,
-            TransactionalPageOutput delegate
-    ) {
-        this.source = source;
-        this.delegate = delegate;
-        this.taskQueue = new LinkedBlockingQueue<>();
-        this.executorService = Executors.newSingleThreadExecutor(
-                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getPluginCode(), taskIndex)).build()
-        );
-        this.result = executorService.submit(new Worker());
-    }
-
-    void add(Page page) {
-        taskQueue.add(() -> {
-            delegate.add(page);
-            return null;
-        });
-    }
-
-    void finish() {
-        taskQueue.add(() -> {
-            delegate.finish();
-            return null;
-        });
-    }
-
-    void close() {
-        taskQueue.add(() -> {
-            delegate.close();
-            return null;
-        });
-    }
-
-    void abort() {
-        taskQueue.add(() -> {
-            delegate.abort();
-            return null;
-        });
-    }
-
-    TaskReport commit() {
-        taskQueue.add(delegate::commit);
-        try {
-            return result.get();
-        } catch (InterruptedException e) {
-            Thread.currentThread().interrupt();
-            throw new RuntimeException(e);
-        } catch (ExecutionException e) {
-            throw new PluginExecutionException(source, e.getCause());
-        } finally {
-            executorService.shutdown();
-        }
-    }
-
-    private class Worker implements Callable<TaskReport> {
-        @Override
-        public TaskReport call() throws InterruptedException {
-            while (true) {
-                final Object result = taskQueue.take().get();
-                if (result instanceof TaskReport) {
-                    return (TaskReport) result;
-                }
-            }
-        }
-    }
-}