embulk-output-multi 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdd6eeb1ca104b3fca01e57c679d0d52c5e483a2
-  data.tar.gz: 03fdeb08212e6769baa2652f149d2e2b27a97f81
+  metadata.gz: bd9a500d9ed46b10465db1c1521c60324a90526b
+  data.tar.gz: fc26433a4aabf02ad3e474c503284801e816c0d8
 SHA512:
-  metadata.gz: 4536fd0e3f5c31acbdcd78931282b776dbe6842f7c5e26ecaeb5c0bd899cf85d9979db6453f6bc8aa77277a0b0d6ec4aa58748e6c95e35a382de739bbe60bebe
-  data.tar.gz: 890c8205e4b6ab4970bdfcb5b2faa9dc47dc5bd367a1c0cc6603c2dc9d4c4dab17724b2064d2bfbef7e078a0063e9971d31971f774f07dd3db7486ac97d467e1
+  metadata.gz: f65a6b68738a9af5638550666c3340d30f8e75fcbe7b142843a977492c98179c4d7f5c1dd875041771608e7e72c4f57113432ca55695b5f07bd378a191446cdd
+  data.tar.gz: 3a2cdbec13b5e9a507fc348ed69ef3270e5fbd90275e03ba1797713e12d609c489eb1dd626592d4bd13aa66879c6bc2cd771343dc292a4862557e65e7f91eb22

data/README.md

@@ -2,6 +2,12 @@
 
 This plugin can copies an output to multiple destinations.
 
+### Notes
+- It's still very experimental version, so might change its configuration names or styles without notification. 
+- It might have performance issues or bugs when loading large records.
+- It might not working on other executors than LocalExecutor.
+- I would appreciate it if you use this and give me reports/feedback!
+
 ## Overview
 
 * **Plugin type**: output
@@ -12,12 +18,13 @@ This plugin can copies an output to multiple destinations.
 ## Configuration
 
 - **outputs**: Configuration of output plugins (array, required)
+- **stop_on_failed_output**: Set true if you want to immediately stop outputting of all plugins with a plugin's error (boolean, default "false")
 
 ## Example
 
 ```yaml
 out:
-  type: copy
+  type: multi
   outputs:
     # Output to stdout
     - type: stdout
@@ -34,6 +41,8 @@ out:
       formatter:
         type: csv
         delimiter: "\t"
+    # And any outputs you want..
+    - type: ...
 ```
 
 ## Build

data/build.gradle

@@ -19,7 +19,7 @@ configurations {
 dependencies {
     testImplementation "org.embulk:embulk-standards:0.9.15"
     testImplementation "org.embulk:embulk-test:0.9.15"
-    testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.1"
+    testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.2"
     testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
     testImplementation "org.junit.jupiter:junit-jupiter-engine:5.3.2"
 }

data/gradle.properties

@@ -1 +1 @@
-version=0.1.0
+version=0.1.1

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -9,7 +9,7 @@ com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
 com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
 com.fasterxml.jackson.module:jackson-module-guice:2.6.7
-com.github.kamatama41:embulk-test-helpers:0.7.1
+com.github.kamatama41:embulk-test-helpers:0.7.2
 com.google.code.findbugs:annotations:3.0.0
 com.google.guava:guava:18.0
 com.google.inject.extensions:guice-multibindings:4.0

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -39,6 +39,10 @@ public class MultiOutputPlugin implements OutputPlugin {
         @Config("outputs")
         List<ConfigSource> getOutputConfigs();
 
+        @Config("stop_on_failed_output")
+        @ConfigDefault("false")
+        boolean getStopOnFailedOutput();
+
         @Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
         @ConfigDefault("null")
         Optional<List<ConfigDiff>> getOutputConfigDiffs();
@@ -95,8 +99,8 @@ public class MultiOutputPlugin implements OutputPlugin {
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
         final PluginTask task = taskSource.loadTask(PluginTask.class);
         final ExecSession session = Exec.session();
-        final List<TransactionalPageOutput> delegates = mapWithPluginDelegate(task, session, delegate ->
-                delegate.open(schema, taskIndex)
+        final List<TransactionalPageOutputDelegate> delegates = mapWithPluginDelegate(task, session, delegate ->
+                new TransactionalPageOutputDelegate(delegate, delegate.open(schema, taskIndex), task.getStopOnFailedOutput())
         );
 
         return new TransactionalPageOutput() {
@@ -267,6 +271,68 @@ public class MultiOutputPlugin implements OutputPlugin {
         }
     }
 
+    private static class TransactionalPageOutputDelegate implements TransactionalPageOutput {
+        private final OutputPluginDelegate source;
+        private final TransactionalPageOutput delegate;
+        private final boolean isStopOnFailedOutput;
+
+        TransactionalPageOutputDelegate(
+                OutputPluginDelegate source, TransactionalPageOutput delegate, boolean isStopOnFailedOutput) {
+            this.source = source;
+            this.delegate = delegate;
+            this.isStopOnFailedOutput = isStopOnFailedOutput;
+        }
+
+        @Override
+        public void add(Page page) {
+            try {
+                delegate.add(page);
+            } catch (Exception e) {
+                warnOrException(e);
+            }
+        }
+
+        @Override
+        public void finish() {
+            try {
+                delegate.finish();
+            } catch (Exception e) {
+                warnOrException(e);
+            }
+        }
+
+        @Override
+        public void close() {
+            try {
+                delegate.close();
+            } catch (Exception e) {
+                warnOrException(e);
+            }
+        }
+
+        @Override
+        public void abort() {
+            try {
+                delegate.abort();
+            } catch (Exception e) {
+                warnOrException(e);
+            }
+        }
+
+        @Override
+        public TaskReport commit() {
+            return delegate.commit();
+        }
+
+        private void warnOrException(Exception e) {
+            final String message = String.format("Failed on output for %s.", source.getPluginNameForLogging());
+            if (isStopOnFailedOutput) {
+                throw new RuntimeException(message, e);
+            }
+            LOGGER.warn(message);
+        }
+    }
+
     private static class ControlDelegate implements OutputPlugin.Control {
         private final int pluginIndex;
         private final RunControlTask controlTask;

data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

@@ -1,6 +1,7 @@
 package org.embulk.output.multi;
 
 import org.embulk.config.ConfigSource;
+import org.embulk.exec.PartialExecutionException;
 import org.embulk.test.EmbulkPluginTest;
 import org.embulk.test.EmbulkTest;
 import org.embulk.test.Record;
@@ -22,6 +23,9 @@ import static org.embulk.test.LocalObjectOutputPlugin.assertRecords;
 import static org.embulk.test.Utils.configFromResource;
 import static org.embulk.test.Utils.record;
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
 
 @EmbulkTest(MultiOutputPlugin.class)
 class TestMultiOutputPlugin extends EmbulkPluginTest {
@@ -111,6 +115,53 @@ class TestMultiOutputPlugin extends EmbulkPluginTest {
         assertRecords(records[3]);
     }
 
+    @Test
+    void testStopOnFailedOutputIsFalse() throws IOException {
+        final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
+        final ConfigSource outConfig = configFromResource("yaml/out_stop_on_failed_output.yml");
+        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(outputStream));
+
+        // Run Embulk without error
+        runOutput(inConfig, outConfig);
+
+        // All records should be loaded for local_object output plugin.
+        final Record[] values = new Record[]{record(1, "user1", 20), record(2, "user2", 21)};
+        assertRecords(values);
+
+        // No record should be loaded for stdout output plugin
+        final BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new ByteArrayInputStream(outputStream.toByteArray()))
+        );
+        assertNull(reader.readLine());
+    }
+
+    @Test
+    void testStopOnFailedOutputIsTrue() throws IOException {
+        final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
+        final ConfigSource outConfig = configFromResource("yaml/out_stop_on_failed_output.yml")
+                .set("stop_on_failed_output", true);
+        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(outputStream));
+
+        // Run Embulk
+        try {
+            runOutput(inConfig, outConfig);
+            fail("No exception");
+        } catch (PartialExecutionException e) {
+            assertTrue(e.getMessage().contains("Failed on output for throw_exception output plugin (pluginIndex: 0)"));
+        }
+
+        // No record should be loaded for local_object output plugin.
+        assertRecords();
+
+        // No record should be loaded for stdout output plugin
+        final BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new ByteArrayInputStream(outputStream.toByteArray()))
+        );
+        assertNull(reader.readLine());
+    }
+
     private static Object[][][] generateValues(int size) {
         Object[][][] values = new Object[size][1][3];
         for (int i = 0; i < size; i++) {

data/src/test/resources/yaml/out_stop_on_failed_output.yml

@@ -0,0 +1,9 @@
+type: multi
+outputs:
+  - type: throw_exception
+    thrown_on: ADD
+    source:
+      type: stdout
+  - type: local_object
+    incremental: true
+    incremental_column: id

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-multi
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Shinichi ISHIMURA
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-02 00:00:00.000000000 Z
+date: 2019-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -50,7 +50,7 @@ files:
 - LICENSE
 - README.md
 - build.gradle
-- classpath/embulk-output-multi-0.1.0.jar
+- classpath/embulk-output-multi-0.1.1.jar
 - gradle.properties
 - gradle/dependency-locks/compileClasspath.lockfile
 - gradle/dependency-locks/testCompileClasspath.lockfile
@@ -64,6 +64,7 @@ files:
 - src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
 - src/test/resources/yaml/in_base.yml
 - src/test/resources/yaml/out_base.yml
+- src/test/resources/yaml/out_stop_on_failed_output.yml
 homepage: https://github.com/kamatama41/embulk-output-multi
 licenses:
 - MIT