embulk-output-multi 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 40d2c979fbce0853ed920cae18f0f867284e9491
-  data.tar.gz: e02bef302e6b8061f9029dc4d7077a00b7563623
+  metadata.gz: 133f0b4290cb49bc787590e22edacf2e4f4df4e1
+  data.tar.gz: bb8c9a01232bb0ab43bc845af45a66039988133d
 SHA512:
-  metadata.gz: ba665f2f4108d65c1cf48ad5a94789e6d719443bb0e8bc7f579f05357407918ab275b41cd55aa96af0ae0854e572427f55039a8502e332c8ab22d6707d6118f9
-  data.tar.gz: 7fa76fd0c1a8c06e89b2b0eff90e6b16e00985db269444a37812da6af0f40e3a74a748b00c7e3dfc675b35f7c60875e5f57efd87bf6963eacf57f05cc04a9e79
+  metadata.gz: 7a63b03b9c0d566b49c88fb666390d94ddab230ef21d1fe4254f7131fc4fb6977610b6b95d9966e98631ec8d842dbdcbfc1639d076f6f7bd394f076a4dbe913d
+  data.tar.gz: 754f5fe52bb5b00f87eebdbaef36157dc4d1e9c1ddcbc541f66d9d6cb54af134c02010a50684f5114a3107ea2dfb2f3ec087807da5aa2b411e13bbd2d1889f7a

data/gradle.properties

@@ -1 +1 @@
-version=0.2.1
+version=0.2.2

data/src/main/java/org/embulk/output/multi/AsyncRunControl.java

@@ -5,22 +5,22 @@ import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.OutputPlugin;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
-import java.util.concurrent.atomic.AtomicReferenceArray;
 
 class AsyncRunControl {
     private static final String THREAD_NAME_FORMAT = "multi-run-control-%d";
     private final MultiOutputPlugin.PluginTask task;
     private final OutputPlugin.Control control;
     private final CountDownLatch latch;
-    private final AtomicReferenceArray<TaskSource> taskSources;
+    private final ConcurrentMap<String, TaskSource> taskSources;
     private final ExecutorService executorService;
     private Future<List<TaskReport>> result;
 
@@ -32,7 +32,7 @@ class AsyncRunControl {
         this.task = task;
         this.control = control;
         this.latch = new CountDownLatch(task.getOutputConfigs().size());
-        this.taskSources = new AtomicReferenceArray<>(task.getOutputConfigs().size());
+        this.taskSources = new ConcurrentHashMap<>(task.getOutputConfigs().size());
         this.executorService = Executors.newSingleThreadExecutor(
                 new ThreadFactoryBuilder().setNameFormat(THREAD_NAME_FORMAT).build()
         );
@@ -42,8 +42,8 @@ class AsyncRunControl {
         result.cancel(true);
     }
 
-    void addTaskSource(int index, TaskSource taskSource) {
-        taskSources.set(index, taskSource);
+    void addTaskSource(String tag, TaskSource taskSource) {
+        taskSources.putIfAbsent(tag, taskSource);
         latch.countDown();
     }
 
@@ -64,10 +64,6 @@ class AsyncRunControl {
         @Override
         public List<TaskReport> call() throws Exception {
             latch.await();
-            List<TaskSource> taskSources = new ArrayList<>(AsyncRunControl.this.taskSources.length());
-            for (int i = 0; i < AsyncRunControl.this.taskSources.length(); i++) {
-                taskSources.add(AsyncRunControl.this.taskSources.get(i));
-            }
             task.setTaskSources(taskSources);
             return control.run(task.dump());
         }

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -16,12 +16,15 @@ import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Page;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
-import java.util.concurrent.Future;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -32,12 +35,13 @@ public class MultiOutputPlugin implements OutputPlugin {
 
         @Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
         @ConfigDefault("null")
-        Optional<List<ConfigDiff>> getOutputConfigDiffs();
+        Optional<Map<String, ConfigDiff>> getOutputConfigDiffs();
 
-        List<TaskSource> getTaskSources();
-        void setTaskSources(List<TaskSource> taskSources);
+        Map<String, TaskSource> getTaskSources();
+        void setTaskSources(Map<String, TaskSource> taskSources);
     }
 
+    private static final Logger LOGGER = LoggerFactory.getLogger(MultiOutputPlugin.class);
     private static final String CONFIG_NAME_OUTPUT_CONFIG_DIFFS = "output_config_diffs";
     static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
 
@@ -123,11 +127,11 @@ public class MultiOutputPlugin implements OutputPlugin {
             @Override
             public TaskReport commit() {
                 final TaskReport report = Exec.newTaskReport();
-                final List<TaskReport> reports = new ArrayList<>();
+                final Map<String, TaskReport> reports = new HashMap<>();
                 final List<OutputPluginDelegate> errorPlugins = new ArrayList<>();
                 for (TransactionalPageOutputDelegate output : delegates) {
                     try {
-                        reports.add(output.commit());
+                        reports.put(output.getTag(), output.commit());
                     } catch (PluginExecutionException e) {
                         errorPlugins.add(e.getPlugin());
                     }
@@ -135,7 +139,7 @@ public class MultiOutputPlugin implements OutputPlugin {
                 if (!errorPlugins.isEmpty()) {
                     throw new RuntimeException(
                             String.format("Following plugins failed to output [%s]",
-                                    errorPlugins.stream().map(OutputPluginDelegate::getPluginCode).collect(Collectors.joining(", "))
+                                    errorPlugins.stream().map(OutputPluginDelegate::getTag).collect(Collectors.joining(", "))
                             ));
                 }
                 report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
@@ -144,12 +148,12 @@ public class MultiOutputPlugin implements OutputPlugin {
         };
     }
 
-    private static ConfigDiff buildConfigDiff(List<Future<ConfigDiff>> runPluginTasks) {
+    private static ConfigDiff buildConfigDiff(List<OutputPluginDelegate.Transaction> transactions) {
         final ConfigDiff configDiff = Exec.newConfigDiff();
-        List<ConfigDiff> configDiffs = new ArrayList<>();
-        for (Future<ConfigDiff> pluginTask : runPluginTasks) {
+        Map<String, ConfigDiff> configDiffs = new HashMap<>();
+        for (OutputPluginDelegate.Transaction transaction: transactions) {
             try {
-                configDiffs.add(pluginTask.get());
+                configDiffs.put(transaction.getTag(), transaction.getResult());
             } catch (InterruptedException e) {
                 Thread.currentThread().interrupt();
                 throw new RuntimeException(e);
@@ -163,18 +167,29 @@ public class MultiOutputPlugin implements OutputPlugin {
 
     private <T> List<T> mapWithPluginDelegate(PluginTask task, ExecSession session, Function<OutputPluginDelegate, T> action) {
         List<T> result = new ArrayList<>();
-        for (int pluginIndex = 0; pluginIndex < task.getOutputConfigs().size(); pluginIndex++) {
-            final ConfigSource config = task.getOutputConfigs().get(pluginIndex);
-            if (task.getOutputConfigDiffs().isPresent()) {
-                config.merge(task.getOutputConfigDiffs().get().get(pluginIndex));
-            }
+        for (int i = 0; i < task.getOutputConfigs().size(); i++) {
+            final ConfigSource config = task.getOutputConfigs().get(i);
             final PluginType pluginType = config.get(PluginType.class, "type");
             final OutputPlugin outputPlugin = session.newPlugin(OutputPlugin.class, pluginType);
+
+            final String tag = String.format("%s_%d", pluginType.getName(), i);
+
+            // Merge ConfigDiff if exists
+            if (task.getOutputConfigDiffs().isPresent()) {
+                final ConfigDiff configDiff = task.getOutputConfigDiffs().get().get(tag);
+                if (configDiff != null) {
+                    config.merge(configDiff);
+                } else {
+                    LOGGER.debug("ConfigDiff for '{}' not found.", tag);
+                }
+            }
+            // Set TaskSource if exists
             TaskSource taskSource = null;
             if (task.getTaskSources() != null) {
-                taskSource = task.getTaskSources().get(pluginIndex);
+                taskSource = task.getTaskSources().get(tag);
             }
-            result.add(action.apply(new OutputPluginDelegate(pluginIndex, pluginType, outputPlugin, config, taskSource)));
+
+            result.add(action.apply(new OutputPluginDelegate(tag, outputPlugin, config, taskSource)));
         }
         return result;
     }

data/src/main/java/org/embulk/output/multi/OutputPluginDelegate.java

@@ -5,7 +5,6 @@ import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.plugin.PluginType;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.Schema;
 import org.embulk.spi.TransactionalPageOutput;
@@ -23,105 +22,92 @@ import java.util.concurrent.Future;
 class OutputPluginDelegate {
     private static final Logger LOGGER = LoggerFactory.getLogger(OutputPluginDelegate.class);
     private static final String THREAD_NAME_FORMAT = "multi-output-plugin-%s-%%d";
-    private final int pluginIndex;
-    private final PluginType type;
+    private final String tag;
     private final OutputPlugin plugin;
     private final ConfigSource config;
     private final TaskSource taskSource;
     private final ExecutorService executorService;
 
     OutputPluginDelegate(
-            int pluginIndex,
-            PluginType type,
+            String tag,
             OutputPlugin plugin,
             ConfigSource config,
             TaskSource taskSource
     ) {
-        this.pluginIndex = pluginIndex;
-        this.type = type;
+        this.tag = tag;
         this.plugin = plugin;
         this.config = config;
         this.taskSource = taskSource;
         this.executorService = Executors.newSingleThreadExecutor(
-                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, generatePluginCode(type, pluginIndex))).build()
+                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, tag)).build()
         );
     }
 
-    Future<ConfigDiff> transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
-        return executorService.submit(() -> {
+    Transaction transaction(Schema schema, int taskCount, AsyncRunControl runControl) {
+        return new Transaction(executorService.submit(() -> {
             try {
-                LOGGER.debug("Run #transaction for {}", getPluginCode());
-                return plugin.transaction(config, schema, taskCount, new Control(pluginIndex, runControl));
+                LOGGER.debug("Run #transaction for {}", getTag());
+                return plugin.transaction(config, schema, taskCount, new Control(runControl));
             } catch (CancellationException e) {
-                LOGGER.error("Canceled #transaction for {} by other plugin's error", getPluginCode());
+                LOGGER.error("Canceled #transaction for {} by other plugin's error", getTag());
                 throw e;
             } catch (Exception e) {
-                LOGGER.error("Transaction for {} failed.", getPluginCode(), e);
+                LOGGER.error("Transaction for {} failed.", getTag(), e);
                 runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
             }
-        });
+        }));
     }
 
-    Future<ConfigDiff> resume(Schema schema, int taskCount, AsyncRunControl runControl) {
-        return executorService.submit(() -> {
+    Transaction resume(Schema schema, int taskCount, AsyncRunControl runControl) {
+        return new Transaction(executorService.submit(() -> {
             try {
-                LOGGER.debug("Run #resume for {}", getPluginCode());
-                return plugin.resume(taskSource, schema, taskCount, new Control(pluginIndex, runControl));
+                LOGGER.debug("Run #resume for {}", getTag());
+                return plugin.resume(taskSource, schema, taskCount, new Control(runControl));
             } catch (CancellationException e) {
-                LOGGER.error("Canceled #resume for {} by other plugin's error", getPluginCode());
+                LOGGER.error("Canceled #resume for {} by other plugin's error", getTag());
                 throw e;
             } catch (Exception e) {
-                LOGGER.error("Resume for {} failed.", getPluginCode(), e);
+                LOGGER.error("Resume for {} failed.", getTag(), e);
                 runControl.cancel();
                 throw e;
             } finally {
                 executorService.shutdown();
             }
-        });
+        }));
     }
 
     void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
-        LOGGER.debug("Run #cleanup for {}", getPluginCode());
+        LOGGER.debug("Run #cleanup for {}", getTag());
         List<TaskReport> successReportsForPlugin = new ArrayList<>();
         for (TaskReport successTaskReport : successTaskReports) {
-            final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+            final TaskReport report = successTaskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
             successReportsForPlugin.add(report);
         }
         plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
     }
 
     TransactionalPageOutput open(Schema schema, int taskIndex) {
-        LOGGER.debug("Run #open for {}", getPluginCode());
+        LOGGER.debug("Run #open for {}", getTag());
         return plugin.open(taskSource, schema, taskIndex);
     }
 
-    PluginType getType() {
-        return type;
+    String getTag() {
+        return tag;
     }
 
-    String getPluginCode() {
-        return generatePluginCode(type, pluginIndex);
-    }
-
-    private static String generatePluginCode(PluginType type, int pluginIndex) {
-        return String.format("%s-%d", type.getName(), pluginIndex);
-    }
-
-    private static class Control implements OutputPlugin.Control {
-        private final int pluginIndex;
+    private class Control implements OutputPlugin.Control {
         private final AsyncRunControl runControl;
 
-        Control(int index, AsyncRunControl runControl) {
-            this.pluginIndex = index;
+        Control(AsyncRunControl runControl) {
             this.runControl = runControl;
         }
 
         @Override
         public List<TaskReport> run(TaskSource taskSource) {
-            runControl.addTaskSource(pluginIndex, taskSource);
+            runControl.addTaskSource(tag, taskSource);
             List<TaskReport> reports;
             try {
                 reports = runControl.waitAndGetResult();
@@ -134,10 +120,26 @@ class OutputPluginDelegate {
 
             final List<TaskReport> result = new ArrayList<>();
             for (TaskReport taskReport : reports) {
-                final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+                final TaskReport report = taskReport.get(TaskReports.class, MultiOutputPlugin.CONFIG_NAME_OUTPUT_TASK_REPORTS).get(tag);
                 result.add(report);
             }
             return result;
         }
     }
+
+    class Transaction {
+        private final Future<ConfigDiff> future;
+
+        private Transaction(Future<ConfigDiff> future) {
+            this.future = future;
+        }
+
+        String getTag() {
+            return OutputPluginDelegate.this.getTag();
+        }
+
+        ConfigDiff getResult() throws ExecutionException, InterruptedException {
+            return future.get();
+        }
+    }
 }

data/src/main/java/org/embulk/output/multi/TaskReports.java

@@ -4,22 +4,22 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import org.embulk.config.TaskReport;
 
-import java.util.List;
+import java.util.Map;
 
 class TaskReports {
-    private final List<TaskReport> reports;
+    private final Map<String, TaskReport> reports;
 
     @JsonCreator
-    TaskReports(@JsonProperty("reports") List<TaskReport> reports) {
+    TaskReports(@JsonProperty("reports") Map<String, TaskReport> reports) {
         this.reports = reports;
     }
 
     @JsonProperty("reports")
-    List<TaskReport> getReports() {
+    Map<String, TaskReport> getReports() {
         return reports;
     }
 
-    TaskReport get(int index) {
-        return reports.get(index);
+    TaskReport get(String tag) {
+        return reports.get(tag);
     }
 }

data/src/main/java/org/embulk/output/multi/TransactionalPageOutputDelegate.java

@@ -31,7 +31,7 @@ class TransactionalPageOutputDelegate {
         this.delegate = delegate;
         this.taskQueue = new LinkedBlockingQueue<>();
         this.executorService = Executors.newSingleThreadExecutor(
-                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getPluginCode(), taskIndex)).build()
+                new ThreadFactoryBuilder().setNameFormat(String.format(THREAD_NAME_FORMAT, source.getTag(), taskIndex)).build()
         );
         this.result = executorService.submit(new Worker());
     }
@@ -78,6 +78,10 @@ class TransactionalPageOutputDelegate {
         }
     }
 
+    String getTag() {
+        return source.getTag();
+    }
+
     private class Worker implements Callable<TaskReport> {
         @Override
         public TaskReport call() throws InterruptedException {

data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

@@ -24,10 +24,10 @@ import static org.embulk.test.Utils.configFromResource;
 import static org.embulk.test.Utils.record;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertNull;
-import static org.junit.jupiter.api.Assertions.assertTrue;
 
 @EmbulkTest(MultiOutputPlugin.class)
 class TestMultiOutputPlugin extends EmbulkPluginTest {
+
     @Test
     void testMultipleOutputWorking() throws IOException {
         final ConfigSource inConfig = configFromResource("yaml/in_base.yml");

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-multi
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Shinichi ISHIMURA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-07 00:00:00.000000000 Z
+date: 2019-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-output-multi-0.2.1.jar
+- classpath/embulk-output-multi-0.2.2.jar
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile