embulk-output-multi 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cdd6eeb1ca104b3fca01e57c679d0d52c5e483a2
+  data.tar.gz: 03fdeb08212e6769baa2652f149d2e2b27a97f81
+SHA512:
+  metadata.gz: 4536fd0e3f5c31acbdcd78931282b776dbe6842f7c5e26ecaeb5c0bd899cf85d9979db6453f6bc8aa77277a0b0d6ec4aa58748e6c95e35a382de739bbe60bebe
+  data.tar.gz: 890c8205e4b6ab4970bdfcb5b2faa9dc47dc5bd367a1c0cc6603c2dc9d4c4dab17724b2064d2bfbef7e078a0063e9971d31971f774f07dd3db7486ac97d467e1

data/.circleci/config.yml

@@ -0,0 +1,42 @@
+version: 2
+jobs:
+  build:
+    docker:
+      - image: openjdk:8
+    working_directory: ~/embulk-filter-hash
+    steps:
+      - checkout
+      - restore_cache:
+          keys:
+            - v1-gradle-{{ checksum "gradle/dependency-locks/compileClasspath.lockfile" }}-{{ checksum "gradle/dependency-locks/testCompileClasspath.lockfile" }}
+            - v1-gradle-
+      - run: ./gradlew check --info
+      - run:
+          name: Save test results
+          command: |
+            mkdir -p ~/junit/
+            find . -type f -regex ".*/build/test-results/.*xml" -exec cp {} ~/junit/ \;
+          when: always
+      - store_test_results:
+          path: ~/junit
+      - store_artifacts:
+          path: ~/junit
+
+      - save_cache:
+          paths:
+            - "~/.gradle"
+            - "~/.m2"
+          key: v1-gradle-{{ checksum "gradle/dependency-locks/compileClasspath.lockfile" }}-{{ checksum "gradle/dependency-locks/testCompileClasspath.lockfile" }}
+
+      - deploy:
+          name: Push Gem to RubyGems.org and bump up
+          command: |
+            if [ "${CIRCLE_BRANCH}" == "release" ]; then
+              mkdir -p ~/.gem
+              curl -f -u ${RUBYGEMS_USER}:${RUBYGEMS_PASSWORD} https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
+              git config --global user.email "shiketaudonko41@gmail.com"
+              git config --global user.name "kamatama41"
+              git checkout master
+              git reset --hard origin/master
+              ./gradlew release -Prelease.useAutomaticVersion=true
+            fi

data/.gitignore

@@ -0,0 +1,13 @@
+*~
+/pkg/
+/tmp/
+*.gemspec
+.gradle/
+/classpath/
+build/
+.idea
+*.iml
+/.settings/
+/.metadata/
+.classpath
+.project

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2019 Shinichi Ishimura
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,43 @@
+# Multi output plugin for Embulk
+
+This plugin can copies an output to multiple destinations.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: no
+* **Resume supported**: yes
+* **Cleanup supported**: yes
+
+## Configuration
+
+- **outputs**: Configuration of output plugins (array, required)
+
+## Example
+
+```yaml
+out:
+  type: copy
+  outputs:
+    # Output to stdout
+    - type: stdout
+    # Output to file as CSV
+    - type: file
+      path_prefix: out_file_
+      file_ext: csv
+      formatter:
+        type: csv
+    # Output to file as TSV
+    - type: file
+      path_prefix: out_file_
+      file_ext: tsv
+      formatter:
+        type: csv
+        delimiter: "\t"
+```
+
+## Build
+
+```
+$ ./gradlew gem  # -t to watch change of files and rebuild continuously
+```

data/build.gradle

@@ -0,0 +1,44 @@
+plugins {
+    id "com.github.kamatama41.embulk" version "0.4.0"
+    id "net.researchgate.release" version "2.8.0"
+}
+
+sourceCompatibility = 1.8
+targetCompatibility = 1.8
+
+repositories {
+    maven { url 'http://kamatama41.github.com/maven-repository/repository' }
+}
+
+configurations {
+    [compileClasspath, testCompileClasspath].each {
+        it.resolutionStrategy.activateDependencyLocking()
+    }
+}
+
+dependencies {
+    testImplementation "org.embulk:embulk-standards:0.9.15"
+    testImplementation "org.embulk:embulk-test:0.9.15"
+    testImplementation "com.github.kamatama41:embulk-test-helpers:0.7.1"
+    testImplementation "org.junit.jupiter:junit-jupiter-api:5.3.2"
+    testImplementation "org.junit.jupiter:junit-jupiter-engine:5.3.2"
+}
+
+test {
+    useJUnitPlatform()
+}
+
+embulk {
+    version = "0.9.15"
+    category = "output"
+    name = "multi"
+    authors = ["Shinichi ISHIMURA"]
+    email = "shiketaudonko41@gmail.com"
+    description = "Dump records into multiple outputs"
+    homepage = "https://github.com/kamatama41/embulk-output-multi"
+}
+
+release {
+    git { requireBranch = 'master' }
+}
+afterReleaseBuild.dependsOn gemPush

data/gradle.properties

@@ -0,0 +1 @@
+version=0.1.0

data/gradle/dependency-locks/compileClasspath.lockfile

@@ -0,0 +1,35 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+aopalliance:aopalliance:1.0
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
+com.fasterxml.jackson.module:jackson-module-guice:2.6.7
+com.google.code.findbugs:annotations:3.0.0
+com.google.guava:guava:18.0
+com.google.inject.extensions:guice-multibindings:4.0
+com.google.inject:guice:4.0
+com.ibm.icu:icu4j:54.1.1
+commons-beanutils:commons-beanutils-core:1.8.3
+commons-cli:commons-cli:1.3.1
+commons-collections:commons-collections:3.2.1
+commons-lang:commons-lang:2.4
+io.airlift:slice:0.9
+io.netty:netty-buffer:4.0.44.Final
+io.netty:netty-common:4.0.44.Final
+javax.inject:javax.inject:1
+javax.validation:validation-api:1.1.0.Final
+joda-time:joda-time:2.9.2
+org.apache.bval:bval-core:0.5
+org.apache.bval:bval-jsr303:0.5
+org.apache.commons:commons-lang3:3.4
+org.apache.velocity:velocity:1.7
+org.embulk:embulk-core:0.9.15
+org.jruby:jruby-complete:9.1.15.0
+org.msgpack:msgpack-core:0.8.11
+org.slf4j:slf4j-api:1.7.12
+org.yaml:snakeyaml:1.18

data/gradle/dependency-locks/testCompileClasspath.lockfile

@@ -0,0 +1,48 @@
+# This is a Gradle generated file for dependency locking.
+# Manual edits can break the build and are not advised.
+# This file is expected to be part of source control.
+aopalliance:aopalliance:1.0
+com.fasterxml.jackson.core:jackson-annotations:2.6.7
+com.fasterxml.jackson.core:jackson-core:2.6.7
+com.fasterxml.jackson.core:jackson-databind:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-guava:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.6.7
+com.fasterxml.jackson.datatype:jackson-datatype-joda:2.6.7
+com.fasterxml.jackson.module:jackson-module-guice:2.6.7
+com.github.kamatama41:embulk-test-helpers:0.7.1
+com.google.code.findbugs:annotations:3.0.0
+com.google.guava:guava:18.0
+com.google.inject.extensions:guice-multibindings:4.0
+com.google.inject:guice:4.0
+com.ibm.icu:icu4j:54.1.1
+commons-beanutils:commons-beanutils-core:1.8.3
+commons-cli:commons-cli:1.3.1
+commons-collections:commons-collections:3.2.1
+commons-lang:commons-lang:2.4
+io.airlift:slice:0.9
+io.netty:netty-buffer:4.0.44.Final
+io.netty:netty-common:4.0.44.Final
+javax.inject:javax.inject:1
+javax.validation:validation-api:1.1.0.Final
+joda-time:joda-time:2.9.2
+junit:junit:4.12
+org.apache.bval:bval-core:0.5
+org.apache.bval:bval-jsr303:0.5
+org.apache.commons:commons-compress:1.10
+org.apache.commons:commons-lang3:3.4
+org.apache.velocity:velocity:1.7
+org.apiguardian:apiguardian-api:1.0.0
+org.embulk:embulk-core:0.9.15
+org.embulk:embulk-standards:0.9.15
+org.embulk:embulk-test:0.9.15
+org.hamcrest:hamcrest-core:1.3
+org.hamcrest:hamcrest-library:1.3
+org.jruby:jruby-complete:9.1.15.0
+org.junit.jupiter:junit-jupiter-api:5.3.2
+org.junit.jupiter:junit-jupiter-engine:5.3.2
+org.junit.platform:junit-platform-commons:1.3.2
+org.junit.platform:junit-platform-engine:1.3.2
+org.msgpack:msgpack-core:0.8.11
+org.opentest4j:opentest4j:1.1.1
+org.slf4j:slf4j-api:1.7.12
+org.yaml:snakeyaml:1.18

data/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.2.1-all.zip

data/gradlew

@@ -0,0 +1,172 @@
+#!/usr/bin/env sh
+
+##############################################################################
+##
+##  Gradle start up script for UN*X
+##
+##############################################################################
+
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
+
+APP_NAME="Gradle"
+APP_BASE_NAME=`basename "$0"`
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD="maximum"
+
+warn () {
+    echo "$*"
+}
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+}
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "`uname`" in
+  CYGWIN* )
+    cygwin=true
+    ;;
+  Darwin* )
+    darwin=true
+    ;;
+  MINGW* )
+    msys=true
+    ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD="$JAVA_HOME/jre/sh/java"
+    else
+        JAVACMD="$JAVA_HOME/bin/java"
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD="java"
+    which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+fi
+
+# Increase the maximum file descriptors if we can.
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
+    MAX_FD_LIMIT=`ulimit -H -n`
+    if [ $? -eq 0 ] ; then
+        if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
+            MAX_FD="$MAX_FD_LIMIT"
+        fi
+        ulimit -n $MAX_FD
+        if [ $? -ne 0 ] ; then
+            warn "Could not set maximum file descriptor limit: $MAX_FD"
+        fi
+    else
+        warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
+    fi
+fi
+
+# For Darwin, add options to specify how the application appears in the dock
+if $darwin; then
+    GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
+fi
+
+# For Cygwin, switch paths to Windows format before running java
+if $cygwin ; then
+    APP_HOME=`cygpath --path --mixed "$APP_HOME"`
+    CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
+
+    # We build the pattern for arguments to be converted via cygpath
+    ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
+    SEP=""
+    for dir in $ROOTDIRSRAW ; do
+        ROOTDIRS="$ROOTDIRS$SEP$dir"
+        SEP="|"
+    done
+    OURCYGPATTERN="(^($ROOTDIRS))"
+    # Add a user-defined pattern to the cygpath arguments
+    if [ "$GRADLE_CYGPATTERN" != "" ] ; then
+        OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
+    fi
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    i=0
+    for arg in "$@" ; do
+        CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
+        CHECK2=`echo "$arg"|egrep -c "^-"`                                 ### Determine if an option
+
+        if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then                    ### Added a condition
+            eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
+        else
+            eval `echo args$i`="\"$arg\""
+        fi
+        i=$((i+1))
+    done
+    case $i in
+        (0) set -- ;;
+        (1) set -- "$args0" ;;
+        (2) set -- "$args0" "$args1" ;;
+        (3) set -- "$args0" "$args1" "$args2" ;;
+        (4) set -- "$args0" "$args1" "$args2" "$args3" ;;
+        (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
+        (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
+        (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
+        (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
+        (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
+    esac
+fi
+
+# Escape application args
+save () {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
+}
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/output/multi.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_output(
+  "multi", "org.embulk.output.multi.MultiOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/settings.gradle

@@ -0,0 +1,10 @@
+/*
+ * This file was generated by the Gradle 'init' task.
+ *
+ * The settings file is used to specify which projects to include in your build.
+ * 
+ * Detailed information about configuring a multi-project build in Gradle can be found
+ * in the user guide at https://docs.gradle.org/4.5.1/userguide/multi_project_builds.html
+ */
+
+rootProject.name = 'embulk-output-multi'

data/src/main/java/org/embulk/output/multi/MultiOutputPlugin.java

@@ -0,0 +1,347 @@
+package org.embulk.output.multi;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.function.Function;
+
+public class MultiOutputPlugin implements OutputPlugin {
+    public interface PluginTask extends Task {
+        @Config("outputs")
+        List<ConfigSource> getOutputConfigs();
+
+        @Config(CONFIG_NAME_OUTPUT_CONFIG_DIFFS)
+        @ConfigDefault("null")
+        Optional<List<ConfigDiff>> getOutputConfigDiffs();
+
+        List<TaskSource> getTaskSources();
+        void setTaskSources(List<TaskSource> taskSources);
+    }
+
+    private static final String CONFIG_NAME_OUTPUT_CONFIG_DIFFS = "output_config_diffs";
+    private static final String CONFIG_NAME_OUTPUT_TASK_REPORTS = "output_task_reports";
+    private static final Logger LOGGER = LoggerFactory.getLogger(MultiOutputPlugin.class);
+    private final ExecutorService executorService;
+
+    public MultiOutputPlugin() {
+        this.executorService = Executors.newCachedThreadPool();
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, Schema schema, int taskCount, OutputPlugin.Control control) {
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        if (task.getOutputConfigs().isEmpty()) {
+            throw new ConfigException("'outputs' must have more than than or equals to 1 element.");
+        }
+        final ExecSession session = Exec.session();
+        final RunControlTask controlTask = new RunControlTask(task, control, executorService);
+        controlTask.runAsynchronously();
+        return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
+                delegate.transaction(schema, taskCount, controlTask)
+        ));
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource, Schema schema, int taskCount, OutputPlugin.Control control) {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final ExecSession session = Exec.session();
+        final RunControlTask controlTask = new RunControlTask(task, control, executorService);
+        controlTask.runAsynchronously();
+        return buildConfigDiff(mapWithPluginDelegate(task, session, delegate ->
+                delegate.resume(schema, taskCount, controlTask)
+        ));
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource, Schema schema, int taskCount, List<TaskReport> successTaskReports) {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final ExecSession session = Exec.session();
+        mapWithPluginDelegate(task, session, delegate -> {
+            delegate.cleanup(schema, taskCount, successTaskReports);
+            return null;
+        });
+    }
+
+    @Override
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex) {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final ExecSession session = Exec.session();
+        final List<TransactionalPageOutput> delegates = mapWithPluginDelegate(task, session, delegate ->
+                delegate.open(schema, taskIndex)
+        );
+
+        return new TransactionalPageOutput() {
+            @Override
+            public void add(Page original) {
+                final Buffer originalBuffer = original.buffer();
+                for (TransactionalPageOutput output : delegates) {
+                    final Buffer copiedBuffer = Buffer.wrap(originalBuffer.array());
+                    copiedBuffer.offset(originalBuffer.offset());
+                    copiedBuffer.limit(originalBuffer.limit());
+
+                    final Page copiedPage = Page.wrap(copiedBuffer);
+                    copiedPage.setStringReferences(new ArrayList<>(original.getStringReferences()));
+                    copiedPage.setValueReferences(new ArrayList<>(original.getValueReferences()));
+
+                    output.add(copiedPage);
+                }
+            }
+
+            @Override
+            public void finish() {
+                for (TransactionalPageOutput output : delegates) {
+                    output.finish();
+                }
+            }
+
+            @Override
+            public void close() {
+                for (TransactionalPageOutput output : delegates) {
+                    output.close();
+                }
+            }
+
+            @Override
+            public void abort() {
+                for (TransactionalPageOutput output : delegates) {
+                    output.abort();
+                }
+            }
+
+            @Override
+            public TaskReport commit() {
+                final TaskReport report = Exec.newTaskReport();
+                final List<TaskReport> reports = new ArrayList<>();
+                for (TransactionalPageOutput output : delegates) {
+                    reports.add(output.commit());
+                }
+                report.set(CONFIG_NAME_OUTPUT_TASK_REPORTS, new TaskReports(reports));
+                return report;
+            }
+        };
+    }
+
+    private static class RunControlTask implements Callable<List<TaskReport>> {
+        private final PluginTask task;
+        private final OutputPlugin.Control control;
+        private final ExecutorService executorService;
+        private final CountDownLatch latch;
+        private final TaskSource[] taskSources;
+        private Future<List<TaskReport>> future;
+
+        RunControlTask(PluginTask task, OutputPlugin.Control control, ExecutorService executorService) {
+            this.task = task;
+            this.control = control;
+            this.executorService = executorService;
+            this.latch = new CountDownLatch(task.getOutputConfigs().size());
+            this.taskSources = new TaskSource[task.getOutputConfigs().size()];
+        }
+
+        @Override
+        public List<TaskReport> call() throws Exception {
+            latch.await();
+            task.setTaskSources(Arrays.asList(taskSources));
+            return control.run(task.dump());
+        }
+
+        void runAsynchronously() {
+            future = executorService.submit(this);
+        }
+
+        void cancel() {
+            future.cancel(true);
+        }
+
+        void addTaskSource(int index, TaskSource taskSource) {
+            taskSources[index] = taskSource;
+            latch.countDown();
+        }
+
+        List<TaskReport> waitAndGetResult() throws ExecutionException, InterruptedException {
+            return future.get();
+        }
+    }
+
+    private static class OutputPluginDelegate {
+        private final int pluginIndex;
+        private final PluginType type;
+        private final OutputPlugin plugin;
+        private final ConfigSource config;
+        private final TaskSource taskSource;
+        private final ExecutorService executorService;
+
+        OutputPluginDelegate(
+                int pluginIndex,
+                PluginType type,
+                OutputPlugin plugin,
+                ConfigSource config,
+                TaskSource taskSource,
+                ExecutorService executorService
+        ) {
+            this.pluginIndex = pluginIndex;
+            this.type = type;
+            this.plugin = plugin;
+            this.config = config;
+            this.taskSource = taskSource;
+            this.executorService = executorService;
+        }
+
+        Future<ConfigDiff> transaction(Schema schema, int taskCount, RunControlTask controlTask) {
+            LOGGER.debug("Run transaction for {}", getPluginNameForLogging());
+            return executorService.submit(() -> {
+                try {
+                    return plugin.transaction(config, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+                } catch (CancellationException e) {
+                    LOGGER.error("Canceled transaction for {} by other plugin's error", getPluginNameForLogging());
+                    throw e;
+                } catch (Exception e) {
+                    LOGGER.error("Transaction for {} failed.", getPluginNameForLogging(), e);
+                    controlTask.cancel();
+                    throw e;
+                }
+            });
+        }
+
+        Future<ConfigDiff> resume(Schema schema, int taskCount, RunControlTask controlTask) {
+            LOGGER.debug("Run resume for {}", getPluginNameForLogging());
+            return executorService.submit(() -> {
+                try {
+                    return plugin.resume(taskSource, schema, taskCount, new ControlDelegate(pluginIndex, controlTask));
+                } catch (CancellationException e) {
+                    LOGGER.error("Canceled resume for {} by other plugin's error", getPluginNameForLogging());
+                    throw e;
+                } catch (Exception e) {
+                    LOGGER.error("Resume for {} failed.", getPluginNameForLogging(), e);
+                    controlTask.cancel();
+                    throw e;
+                }
+            });
+        }
+
+        void cleanup(Schema schema, int taskCount, List<TaskReport> successTaskReports) {
+            LOGGER.debug("Run cleanup for {}", getPluginNameForLogging());
+            List<TaskReport> successReportsForPlugin = new ArrayList<>();
+            for (TaskReport successTaskReport : successTaskReports) {
+                final TaskReport report = successTaskReport.get(TaskReports.class, CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+                successReportsForPlugin.add(report);
+                plugin.cleanup(taskSource, schema, taskCount, successReportsForPlugin);
+            }
+        }
+
+        TransactionalPageOutput open(Schema schema, int taskIndex) {
+            LOGGER.debug("Run open for {}", getPluginNameForLogging());
+            return plugin.open(taskSource, schema, taskIndex);
+        }
+
+        private String getPluginNameForLogging() {
+            return String.format("%s output plugin (pluginIndex: %s)", type.getName(), pluginIndex);
+        }
+    }
+
+    private static class ControlDelegate implements OutputPlugin.Control {
+        private final int pluginIndex;
+        private final RunControlTask controlTask;
+
+        ControlDelegate(int index, RunControlTask controlTask) {
+            this.pluginIndex = index;
+            this.controlTask = controlTask;
+        }
+
+        @Override
+        public List<TaskReport> run(TaskSource taskSource) {
+            controlTask.addTaskSource(pluginIndex, taskSource);
+            List<TaskReport> reports;
+            try {
+                reports = controlTask.waitAndGetResult();
+            } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
+            }
+
+            final List<TaskReport> result = new ArrayList<>();
+            for (TaskReport taskReport : reports) {
+                final TaskReport report = taskReport.get(TaskReports.class, CONFIG_NAME_OUTPUT_TASK_REPORTS).get(pluginIndex);
+                result.add(report);
+            }
+            return result;
+        }
+    }
+
+    private static class TaskReports {
+        private final List<TaskReport> reports;
+
+        @JsonCreator
+        TaskReports(@JsonProperty("reports") List<TaskReport> reports) {
+            this.reports = reports;
+        }
+
+        @JsonProperty("reports")
+        List<TaskReport> getReports() {
+            return reports;
+        }
+
+        TaskReport get(int index) {
+            return reports.get(index);
+        }
+    }
+
+    private static ConfigDiff buildConfigDiff(List<Future<ConfigDiff>> runPluginTasks) {
+        final ConfigDiff configDiff = Exec.newConfigDiff();
+        List<ConfigDiff> configDiffs = new ArrayList<>();
+        for (Future<ConfigDiff> pluginTask : runPluginTasks) {
+            try {
+                configDiffs.add(pluginTask.get());
+            } catch (InterruptedException | ExecutionException e) {
+                throw new RuntimeException(e);
+            }
+        }
+        configDiff.set(CONFIG_NAME_OUTPUT_CONFIG_DIFFS, configDiffs);
+        return configDiff;
+    }
+
+    private <T> List<T> mapWithPluginDelegate(PluginTask task, ExecSession session, Function<OutputPluginDelegate, T> action) {
+        List<T> result = new ArrayList<>();
+        for (int pluginIndex = 0; pluginIndex < task.getOutputConfigs().size(); pluginIndex++) {
+            final ConfigSource config = task.getOutputConfigs().get(pluginIndex);
+            if (task.getOutputConfigDiffs().isPresent()) {
+                config.merge(task.getOutputConfigDiffs().get().get(pluginIndex));
+            }
+            final PluginType pluginType = config.get(PluginType.class, "type");
+            final OutputPlugin outputPlugin = session.newPlugin(OutputPlugin.class, pluginType);
+            TaskSource taskSource = null;
+            if (task.getTaskSources() != null) {
+                taskSource = task.getTaskSources().get(pluginIndex);
+            }
+            result.add(action.apply(new OutputPluginDelegate(pluginIndex, pluginType, outputPlugin, config, taskSource, executorService)));
+        }
+        return result;
+    }
+}

data/src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java

@@ -0,0 +1,127 @@
+package org.embulk.output.multi;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkPluginTest;
+import org.embulk.test.EmbulkTest;
+import org.embulk.test.Record;
+import org.embulk.test.TestingEmbulk;
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.Arrays;
+import java.util.Set;
+
+import static java.util.stream.Collectors.joining;
+import static java.util.stream.Collectors.toSet;
+import static org.embulk.test.LocalObjectOutputPlugin.assertRecords;
+import static org.embulk.test.Utils.configFromResource;
+import static org.embulk.test.Utils.record;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@EmbulkTest(MultiOutputPlugin.class)
+class TestMultiOutputPlugin extends EmbulkPluginTest {
+    @Test
+    void testMultipleOutputWorking() throws IOException {
+        final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
+        final ConfigSource outConfig = configFromResource("yaml/out_base.yml");
+        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(outputStream));
+
+        runOutput(inConfig, outConfig);
+
+        // Check local_object output plugin working
+        final Record[] values = new Record[]{record(1, "user1", 20), record(2, "user2", 21)};
+        assertRecords(values);
+
+        // Check stdout output plugin working
+        final BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new ByteArrayInputStream(outputStream.toByteArray()))
+        );
+        // Header
+        assertEquals("id,name,age", reader.readLine());
+        // Values
+        Set<String> expected = Arrays.stream(values)
+                .map(v -> v.getValues().stream().map(Object::toString).collect(joining(",")))
+                .collect(toSet());
+        Set<String> actual = reader.lines().collect(toSet());
+        assertEquals(expected, actual);
+    }
+
+    @Test
+    void testMultipleOutputWorkingWithBigRecords() throws IOException {
+        final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
+        // Set 1000 records
+        Object[][][] values = generateValues(1000);
+        inConfig.set("values", values);
+
+        final ConfigSource outConfig = configFromResource("yaml/out_base.yml");
+        final ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
+        System.setOut(new PrintStream(outputStream));
+
+        runOutput(inConfig, outConfig);
+
+        // Check local_object output plugin working
+        assertRecords(toRecords(values));
+
+        // Check stdout output plugin working
+        final BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new ByteArrayInputStream(outputStream.toByteArray()))
+        );
+        // Header
+        assertEquals("id,name,age", reader.readLine());
+        // Values
+        Set<String> expected = Arrays.stream(values).map(v -> String.format("%s,%s,%s", v[0][0], v[0][1], v[0][2])).collect(toSet());
+        Set<String> actual = reader.lines().collect(toSet());
+        assertEquals(expected, actual);
+    }
+
+    @Test
+    void testOutputConfigDiffs() {
+        final ConfigSource inConfig = configFromResource("yaml/in_base.yml");
+        // Set 3 records
+        Object[][][] values = generateValues(3);
+        inConfig.set("values", values);
+
+        // Set incremental column
+        final ConfigSource outConfig = configFromResource("yaml/out_base.yml");
+
+        outConfig.set("incremental", true);
+        outConfig.set("incremental_column", "id");
+
+        // Run Embulk
+        TestingEmbulk.RunResult result = runOutput(inConfig, outConfig);
+
+        // All records should be loaded.
+        assertRecords(toRecords(values));
+
+        // Set 4 records
+        values = generateValues(4);
+        inConfig.set("values", values);
+
+        // Rerun with config diff
+        runOutput(inConfig, outConfig, result.getConfigDiff());
+
+        // Only 1 records (id=4) should be loaded.
+        final Record[] records = toRecords(values);
+        assertRecords(records[3]);
+    }
+
+    private static Object[][][] generateValues(int size) {
+        Object[][][] values = new Object[size][1][3];
+        for (int i = 0; i < size; i++) {
+            values[i][0][0] = i + 1;               // ID
+            values[i][0][1] = "user" + (i + 1);    // Name
+            values[i][0][2] = 20;                  // Age
+        }
+        return values;
+    }
+
+    private static Record[] toRecords(Object[][][] values) {
+        return Arrays.stream(values).map(v -> record(v[0])).toArray(Record[]::new);
+    }
+}

data/src/test/resources/yaml/in_base.yml

@@ -0,0 +1,8 @@
+type: config
+columns:
+  - {name: id, type: long}
+  - {name: name, type: string}
+  - {name: age, type: long}
+values:
+  - - [1, "user1", 20]
+  - - [2, "user2", 21]

data/src/test/resources/yaml/out_base.yml

@@ -0,0 +1,7 @@
+type: multi
+outputs:
+  - type: stdout
+    prints_column_names: true
+  - type: local_object
+    incremental: true
+    incremental_column: id

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-multi
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Shinichi ISHIMURA
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2019-03-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Dump records into multiple outputs
+email:
+- shiketaudonko41@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- LICENSE
+- README.md
+- build.gradle
+- classpath/embulk-output-multi-0.1.0.jar
+- gradle.properties
+- gradle/dependency-locks/compileClasspath.lockfile
+- gradle/dependency-locks/testCompileClasspath.lockfile
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/output/multi.rb
+- settings.gradle
+- src/main/java/org/embulk/output/multi/MultiOutputPlugin.java
+- src/test/java/org/embulk/output/multi/TestMultiOutputPlugin.java
+- src/test/resources/yaml/in_base.yml
+- src/test/resources/yaml/out_base.yml
+homepage: https://github.com/kamatama41/embulk-output-multi
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.6.8
+signing_key:
+specification_version: 4
+summary: Multi output plugin for Embulk
+test_files: []