embulk 0.7.11-java → 0.8.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63d78296ab5c306a09832af9d838a380399c16a7
-  data.tar.gz: d0a267d4052f88ec276ec8cf0000ce64c07e224e
+  metadata.gz: 55c8a96cd529ab0f895063f6d73c36522a96d0be
+  data.tar.gz: 6d416a8b3f768a941a5a386107d70893afff1509
 SHA512:
-  metadata.gz: e303ef0b199c2ad2246be937103c8c26218635a032e1da7bb6c3798a92eca13c7fdbc91b7612c7b97d15506282b060892db5c4b03bf9206b804bbd3d8ca055ad
-  data.tar.gz: 48b676b557f9be90f915bded8b6589f39671a24584e41989d436a3f7f746dc46aae79977ba364b1bfaef23e3593b842078d6ce00b517a367712c43946b599ba0
+  metadata.gz: fb9a45ea1d54286c3626ab98055783043b439e6a0e34380d3431cc8ef30bb3cb7f7043f70714f2929ef10421acc5bfdf795f05b481a419896a93efa17a5da535
+  data.tar.gz: 8947eb80b213318118325705087655d33c05b53a5861dccfc8d79d0be16f6c01612755bc6daed6a2f17790dcbadbed80d22653877a245ca434f171ca90869658

data/Gemfile.lock

@@ -1,13 +1,13 @@
 PATH
   remote: .
   specs:
-    embulk (0.7.9)
-      jruby-jars (= 9.0.0.0)
+    embulk (0.8.0)
+      jruby-jars (= 9.0.4.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    jruby-jars (9.0.0.0)
+    jruby-jars (9.0.4.0)
     kramdown (1.5.0)
     power_assert (0.2.2)
     rake (10.4.2)

data/README.md

@@ -110,7 +110,7 @@ To use the bundle, add `-b <bundle_dir>` option to `guess`, `preview`, or `run`
 See the generated \<bundle_dir>/Gemfile file how to plugin bundles work.
 
 ```
-embulk mkbundle ./embulk_bundle
+embulk mkbundle ./embulk_bundle  # please edit ./embulk_bundle/Gemfile to add plugins. Detailed usage is written in the Gemfile
 embulk guess -b ./embulk_bundle ...
 embulk run   -b ./embulk_bundle ...
 ```

data/build.gradle

@@ -16,10 +16,10 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.11'
+    version = '0.8.0'
 
     ext {
-        jrubyVersion = '9.0.0.0'
+        jrubyVersion = '9.0.4.0'
     }
 
     apply plugin: 'java'

data/embulk-core/build.gradle

@@ -16,6 +16,7 @@ import com.github.jrubygradle.JRubyPrepare
 
 // determine which dependencies have updates: $ gradle dependencyUpdates
 dependencies {
+    compile 'org.embulk:guice-bootstrap:0.1.1'
     compile 'com.google.guava:guava:18.0'
     compile 'com.google.inject:guice:4.0'
     compile 'com.google.inject.extensions:guice-multibindings:4.0'
@@ -37,6 +38,7 @@ dependencies {
     compile 'joda-time:joda-time:2.8.1'
     compile 'io.netty:netty-buffer:5.0.0.Alpha1'
     compile 'org.fusesource.jansi:jansi:1.11'
+    compile 'org.msgpack:msgpack-core:0.8.1'
 
     // For embulk/guess/charset.rb. See also embulk.gemspec
     compile 'com.ibm.icu:icu4j:54.1.1'

data/embulk-core/src/main/java/org/embulk/config/ConfigLoader.java

@@ -15,6 +15,9 @@ import com.fasterxml.jackson.databind.RuntimeJsonMappingException;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import org.yaml.snakeyaml.Yaml;
+import org.yaml.snakeyaml.DumperOptions;
+import org.yaml.snakeyaml.representer.Representer;
+import org.yaml.snakeyaml.constructor.SafeConstructor;
 
 public class ConfigLoader
 {
@@ -60,7 +63,7 @@ public class ConfigLoader
 
     public ConfigSource fromYamlString(String string)
     {
-        JsonNode node = objectToJson(new Yaml().load(string));
+        JsonNode node = objectToJson(newYaml().load(string));
         validateJsonNode(node);
         return new DataSourceImpl(model, (ObjectNode) node);
     }
@@ -74,7 +77,7 @@ public class ConfigLoader
 
     public ConfigSource fromYaml(InputStream stream) throws IOException
     {
-        JsonNode node = objectToJson(new Yaml().load(stream));
+        JsonNode node = objectToJson(newYaml().load(stream));
         validateJsonNode(node);
         return new DataSourceImpl(model, (ObjectNode) node);
     }
@@ -107,7 +110,7 @@ public class ConfigLoader
     {
         ObjectNode source = new ObjectNode(JsonNodeFactory.instance);
         DataSource ds = new DataSourceImpl(model, source);
-        Yaml yaml = new Yaml();
+        Yaml yaml = newYaml();
         for (Map.Entry<String, String> pair : props.entrySet()) {
             if (!pair.getKey().startsWith(keyPrefix)) {
                 continue;
@@ -138,4 +141,9 @@ public class ConfigLoader
             throw new RuntimeException(ex);
         }
     }
+
+    private Yaml newYaml()
+    {
+        return new Yaml(new SafeConstructor(), new Representer(), new DumperOptions(), new YamlTagResolver());
+    }
 }

data/embulk-core/src/main/java/org/embulk/config/YamlTagResolver.java

@@ -0,0 +1,53 @@
+package org.embulk.config;
+
+import java.util.List;
+import java.util.regex.Pattern;
+import org.yaml.snakeyaml.resolver.Resolver;
+import org.yaml.snakeyaml.nodes.Tag;
+import org.yaml.snakeyaml.nodes.NodeId;
+
+public class YamlTagResolver
+    extends Resolver
+{
+    // Resolver converts a node (scalar, sequence, map, or !!tag with them)
+    // to a tag (INT, FLOAT, STR, SEQ, MAP, ...). For example, converting
+    // "123" (scalar) to 123 (INT), or "true" (scalar) to true (BOOL).
+    // This is called by snakeyaml Composer which converts parser events
+    // into an object.
+    //
+    // jackson-dataformat-yaml doesn't use this because it traverses parser
+    // events without using Composer.
+
+    public static final Pattern FLOAT_EXCEPTING_ZERO_START = Pattern
+        .compile("^([-+]?(\\.[0-9]+|[1-9][0-9_]*(\\.[0-9_]*)?)([eE][-+]?[0-9]+)?|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*|[-+]?\\.(?:inf|Inf|INF)|\\.(?:nan|NaN|NAN))$");
+
+    @Override
+    public void addImplicitResolver(Tag tag, Pattern regexp, String first)
+    {
+        // This method is called by constructor through addImplicitResolvers
+        // to setup default implicit resolvers.
+
+        if (tag.equals(Tag.FLOAT)) {
+            super.addImplicitResolver(Tag.FLOAT, FLOAT_EXCEPTING_ZERO_START, "-+0123456789.");
+        }
+        else if (tag.equals(Tag.BOOL)) {
+            // use stricter rule (reject 'On', 'Off', 'Yes', 'No')
+            super.addImplicitResolver(Tag.BOOL, Pattern.compile("^(?:[Tt]rue|[Ff]alse)$"), "TtFf");
+        }
+        else if (tag.equals(Tag.TIMESTAMP)) {
+            // This solves some unexpected behavior that snakeyaml
+            // deserializes "2015-01-01 00:00:00" to java.util.Date
+            // but jackson serializes java.util.Date to an integer.
+            return;
+        }
+        else {
+            super.addImplicitResolver(tag, regexp, first);
+        }
+    }
+
+    @Override
+    public Tag resolve(NodeId kind, String value, boolean implicit)
+    {
+        return super.resolve(kind, value, implicit);  // checks implicit resolvers
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ExecModule.java

@@ -36,7 +36,6 @@ public class ExecModule
         registerPluginTo(binder, ParserPlugin.class, "system_sampling", SamplingParserPlugin.class);
 
         // LocalExecutorPlugin
-        binder.bind(LocalThreadExecutor.class).in(Scopes.SINGLETON);
         registerPluginTo(binder, ExecutorPlugin.class, "local", LocalExecutorPlugin.class);
 
         // serde

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java

@@ -6,124 +6,534 @@ import java.util.concurrent.Callable;
 import java.util.concurrent.Future;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutionException;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import org.slf4j.Logger;
 import com.google.inject.Inject;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
 import org.embulk.config.TaskReport;
 import org.embulk.spi.Exec;
+import org.embulk.spi.ExecSession;
 import org.embulk.spi.ExecutorPlugin;
 import org.embulk.spi.ProcessTask;
 import org.embulk.spi.ProcessState;
 import org.embulk.spi.Schema;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.AbortTransactionResource;
+import org.embulk.spi.CloseResource;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.plugin.compat.PluginWrappers;
+import org.embulk.spi.util.Filters;
 import org.embulk.spi.util.Executors;
 import org.embulk.spi.util.Executors.ProcessStateCallback;
 
 public class LocalExecutorPlugin
         implements ExecutorPlugin
 {
-    private final ExecutorService executor;
+    private int defaultMaxThreads;
+    private int defaultMinThreads;
 
     @Inject
-    public LocalExecutorPlugin(LocalThreadExecutor executor)
+    public LocalExecutorPlugin(@ForSystemConfig ConfigSource systemConfig)
     {
-        this.executor = executor.getExecutorService();
+        int cores = Runtime.getRuntime().availableProcessors();
+        this.defaultMaxThreads = systemConfig.get(Integer.class, "max_threads", cores * 2);
+        this.defaultMinThreads = systemConfig.get(Integer.class, "min_output_tasks", cores);
     }
 
     @Override
-    public void transaction(ConfigSource config, Schema outputSchema, final int inputTaskCount,
+    public void transaction(ConfigSource config, Schema outputSchema, int inputTaskCount,
             ExecutorPlugin.Control control)
     {
-        control.transaction(outputSchema, inputTaskCount, new Executor() {
-            public void execute(ProcessTask task, ProcessState state)
-            {
-                localExecute(task, inputTaskCount, state);
-            }
-        });
+        try (AbstractLocalExecutor exec = newExecutor(config, inputTaskCount)) {
+            control.transaction(outputSchema, exec.getOutputTaskCount(), exec);
+        }
     }
 
-    private void localExecute(ProcessTask task, int taskCount, ProcessState state)
+    private AbstractLocalExecutor newExecutor(ConfigSource config, int inputTaskCount)
     {
         Logger log = Exec.getLogger(LocalExecutorPlugin.class);
+        int maxThreads = config.get(Integer.class, "max_threads", defaultMaxThreads);
+        int minThreads = config.get(Integer.class, "min_output_tasks", defaultMinThreads);
+        if (inputTaskCount < minThreads) {
+            int scatterCount = (minThreads + inputTaskCount - 1) / inputTaskCount;
+            log.info("Using local thread executor with max_threads={} / output tasks {} = input tasks {} * {}",
+                    maxThreads, inputTaskCount * scatterCount, inputTaskCount, scatterCount);
+            return new ScatterExecutor(maxThreads, inputTaskCount, scatterCount);
+        }
+        else {
+            log.info("Using local thread executor with max_threads={} / tasks={}", maxThreads, inputTaskCount);
+            return new DirectExecutor(maxThreads, inputTaskCount);
+        }
+    }
 
-        state.initialize(taskCount, taskCount);
+    private static abstract class AbstractLocalExecutor
+            implements Executor, AutoCloseable
+    {
+        protected final Logger log = Exec.getLogger(LocalExecutorPlugin.class);
 
-        List<Future<Throwable>> futures = new ArrayList<>(taskCount);
-        try {
-            for (int i=0; i < taskCount; i++) {
-                if (state.getOutputTaskState(i).isCommitted()) {
-                    log.warn("Skipped resumed task {}", i);
-                    futures.add(null);  // resumed
-                } else {
-                    futures.add(startProcessor(task, i, state));
-                }
-            }
-            showProgress(log, state, taskCount);
+        protected final int inputTaskCount;
+        protected final int outputTaskCount;
+
+        public AbstractLocalExecutor(int inputTaskCount, int outputTaskCount)
+        {
+            this.inputTaskCount = inputTaskCount;
+            this.outputTaskCount = outputTaskCount;
+        }
+
+        public int getOutputTaskCount()
+        {
+            return outputTaskCount;
+        }
+
+        @Override
+        public void execute(ProcessTask task, ProcessState state)
+        {
+            state.initialize(inputTaskCount, outputTaskCount);
 
-            for (int i=0; i < taskCount; i++) {
-                if (futures.get(i) == null) {
-                    continue;
+            List<Future<Throwable>> futures = new ArrayList<>(inputTaskCount);
+            try {
+                for (int i = 0; i < inputTaskCount; i++) {
+                    futures.add(startInputTask(task, state, i));
                 }
-                try {
-                    state.getInputTaskState(i).setException(futures.get(i).get());
-                } catch (ExecutionException ex) {
-                    state.getInputTaskState(i).setException(ex.getCause());
-                    //Throwables.propagate(ex.getCause());
-                } catch (InterruptedException ex) {
-                    state.getInputTaskState(i).setException(new ExecutionInterruptedException(ex));
+                showProgress(state, inputTaskCount);
+
+                for (int i = 0; i < inputTaskCount; i++) {
+                    if (futures.get(i) == null) {
+                        continue;
+                    }
+                    try {
+                        state.getInputTaskState(i).setException(futures.get(i).get());
+                    }
+                    catch (ExecutionException ex) {
+                        state.getInputTaskState(i).setException(ex.getCause());
+                        //Throwables.propagate(ex.getCause());
+                    }
+                    catch (InterruptedException ex) {
+                        state.getInputTaskState(i).setException(new ExecutionInterruptedException(ex));
+                    }
+                    showProgress(state, inputTaskCount);
                 }
-                showProgress(log, state, taskCount);
             }
-        } finally {
-            for (Future<Throwable> future : futures) {
-                if (future != null && !future.isDone()) {
-                    future.cancel(true);
-                    // TODO join?
+            finally {
+                for (Future<Throwable> future : futures) {
+                    if (future != null && !future.isDone()) {
+                        future.cancel(true);
+                        // TODO join?
+                    }
                 }
             }
         }
+
+        @Override
+        public abstract void close();
+
+        private void showProgress(ProcessState state, int taskCount)
+        {
+            int started = 0;
+            int finished = 0;
+            for (int i = 0; i < taskCount; i++) {
+                if (state.getOutputTaskState(i).isStarted()) { started++; }
+                if (state.getOutputTaskState(i).isFinished()) { finished++; }
+            }
+
+            log.info(String.format("{done:%3d / %d, running: %d}", finished, taskCount, started - finished));
+        }
+
+        protected abstract Future<Throwable> startInputTask(ProcessTask task, ProcessState state, int taskIndex);
+    }
+
+    public static class DirectExecutor
+            extends AbstractLocalExecutor
+    {
+        protected final ExecutorService executor;
+
+        public DirectExecutor(int maxThreads, int taskCount)
+        {
+            super(taskCount, taskCount);
+            this.executor = java.util.concurrent.Executors.newFixedThreadPool(maxThreads,
+                    new ThreadFactoryBuilder()
+                            .setNameFormat("embulk-executor-%d")
+                            .setDaemon(true)
+                            .build());
+        }
+
+        @Override
+        public void close()
+        {
+            executor.shutdown();
+        }
+
+        @Override
+        protected Future<Throwable> startInputTask(final ProcessTask task, final ProcessState state, final int taskIndex)
+        {
+            if (state.getOutputTaskState(taskIndex).isCommitted()) {
+                log.warn("Skipped resumed task {}", taskIndex);
+                return null;  // resumed
+            }
+
+            return executor.submit(new Callable<Throwable>() {
+                public Throwable call()
+                {
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) {
+                        Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() {
+                            public void started()
+                            {
+                                state.getInputTaskState(taskIndex).start();
+                                state.getOutputTaskState(taskIndex).start();
+                            }
+
+                            public void inputCommitted(TaskReport report)
+                            {
+                                state.getInputTaskState(taskIndex).setTaskReport(report);
+                            }
+
+                            public void outputCommitted(TaskReport report)
+                            {
+                                state.getOutputTaskState(taskIndex).setTaskReport(report);
+                            }
+                        });
+                        return null;
+                    }
+                    finally {
+                        state.getInputTaskState(taskIndex).finish();
+                        state.getOutputTaskState(taskIndex).finish();
+                    }
+                }
+            });
+        }
     }
 
-    private void showProgress(Logger log, ProcessState state, int taskCount)
+    public static class ScatterExecutor
+            extends AbstractLocalExecutor
     {
-        int started = 0;
-        int finished = 0;
-        for (int i=0; i < taskCount; i++) {
-            if (state.getInputTaskState(i).isStarted()) { started++; }
-            if (state.getOutputTaskState(i).isFinished()) { finished++; }
+        private final int scatterCount;
+        private final int inputTaskCount;
+        private final ExecutorService inputExecutor;
+        private final ExecutorService outputExecutor;
+
+        public ScatterExecutor(int maxThreads, int inputTaskCount, int scatterCount)
+        {
+            super(inputTaskCount, inputTaskCount * scatterCount);
+            this.inputTaskCount = inputTaskCount;
+            this.scatterCount = scatterCount;
+            this.inputExecutor = java.util.concurrent.Executors.newFixedThreadPool(
+                    Math.max(maxThreads / scatterCount, 1),
+                    new ThreadFactoryBuilder()
+                            .setNameFormat("embulk-input-executor-%d")
+                            .setDaemon(true)
+                            .build());
+            this.outputExecutor = java.util.concurrent.Executors.newCachedThreadPool(
+                    new ThreadFactoryBuilder()
+                            .setNameFormat("embulk-output-executor-%d")
+                            .setDaemon(true)
+                            .build());
+        }
+
+        @Override
+        public void close()
+        {
+            inputExecutor.shutdown();
+            outputExecutor.shutdown();
         }
 
-        log.info(String.format("{done:%3d / %d, running: %d}", finished, taskCount, started - finished));
+        @Override
+        protected Future<Throwable> startInputTask(final ProcessTask task, final ProcessState state, final int taskIndex)
+        {
+            if(isAllScatterOutputFinished(state, taskIndex)) {
+                log.warn("Skipped resumed input task {}", taskIndex);
+                return null;  // resumed
+            }
+
+            return inputExecutor.submit(new Callable<Throwable>() {
+                public Throwable call()
+                {
+                    try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) {
+                        runInputTask(Exec.session(), task, state, taskIndex);
+                        return null;
+                    }
+                }
+            });
+        }
+
+        private boolean isAllScatterOutputFinished(ProcessState state, int taskIndex) {
+            for (int i = 0; i < scatterCount; i++) {
+                int outputTaskIndex = taskIndex * scatterCount + i;
+                if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private void runInputTask(ExecSession exec, ProcessTask task, ProcessState state, int taskIndex)
+        {
+            InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
+            List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
+            OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
+
+            try (ScatterTransactionalPageOutput tran = new ScatterTransactionalPageOutput(state, taskIndex, scatterCount)) {
+                tran.openOutputs(outputPlugin, task.getOutputSchema(), task.getOutputTaskSource());
+
+                try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
+                    tran.openFilters(filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources());
+
+                    tran.startWorkers(outputExecutor);
+
+                    // started
+                    state.getInputTaskState(taskIndex).start();
+                    for (int i = 0; i < scatterCount; i++) {
+                        state.getOutputTaskState(taskIndex * scatterCount + i).start();
+                    }
+
+                    TaskReport inputTaskReport = inputPlugin.run(task.getInputTaskSource(), task.getInputSchema(), taskIndex, tran);
+
+                    // inputCommitted
+                    if (inputTaskReport == null) {
+                        inputTaskReport = exec.newTaskReport();
+                    }
+                    state.getInputTaskState(taskIndex).setTaskReport(inputTaskReport);
+
+                    // outputCommitted
+                    tran.commit();
+                }
+            }
+            finally {
+                state.getInputTaskState(taskIndex).finish();
+                state.getOutputTaskState(taskIndex).finish();
+            }
+        }
     }
 
-    private Future<Throwable> startProcessor(final ProcessTask task, final int taskIndex, final ProcessState state)
+    private static class ScatterTransactionalPageOutput
+            implements TransactionalPageOutput
     {
-        return executor.submit(new Callable<Throwable>() {
-            public Throwable call()
+        private static final Page DONE_PAGE = Page.allocate(0);
+
+        private static class OutputWorker
+                implements Callable<Throwable>
+        {
+            private final PageOutput output;
+            private final Future<Throwable> future;
+            private boolean done;
+            private Page queued;
+
+            public OutputWorker(PageOutput output, ExecutorService executor)
             {
-                try (SetCurrentThreadName dontCare = new SetCurrentThreadName(String.format("task-%04d", taskIndex))) {
-                    Executors.process(Exec.session(), task, taskIndex, new ProcessStateCallback() {
-                        public void started()
-                        {
-                            state.getInputTaskState(taskIndex).start();
-                            state.getOutputTaskState(taskIndex).start();
-                        }
+                this.output = output;
+                this.done = done;
+                this.future = executor.submit(this);
+            }
 
-                        public void inputCommitted(TaskReport report)
-                        {
-                            state.getInputTaskState(taskIndex).setTaskReport(report);
-                        }
+            public synchronized void add(Page page)
+                    throws InterruptedException
+            {
+                while (true) {
+                    if (queued == null) {
+                        queued = page;
+                        notifyAll();
+                        return;
+                    }
+                    else if (queued == DONE_PAGE) {
+                        page.release();
+                        return;
+                    }
+                    wait();
+                }
+            }
+
+            public Throwable join()
+                    throws InterruptedException
+            {
+                try {
+                    return future.get();
+                }
+                catch (ExecutionException ex) {
+                    return ex.getCause();
+                }
+            }
 
-                        public void outputCommitted(TaskReport report)
-                        {
-                            state.getOutputTaskState(taskIndex).setTaskReport(report);
+            @Override
+            public synchronized Throwable call()
+                    throws InterruptedException
+            {
+                try {
+                    while (true) {
+                        if (queued != null) {
+                            if (queued == DONE_PAGE) {
+                                return null;
+                            }
+                            output.add(queued);
+                            queued = null;
+                            notifyAll();
+                        }
+                        wait();
+                    }
+                }
+                finally {
+                    try {
+                        if (queued != null && queued != DONE_PAGE) {
+                            queued.release();
+                            queued = null;
                         }
-                    });
-                    return null;
-                } finally {
-                    state.getInputTaskState(taskIndex).finish();
-                    state.getOutputTaskState(taskIndex).finish();
+                    }
+                    finally {
+                        queued = DONE_PAGE;
+                    }
+                    notifyAll();
+                }
+            }
+        }
+
+        private final ProcessState state;
+        private final int taskIndex;
+        private final int scatterCount;
+
+        private final TransactionalPageOutput[] trans;
+        private final PageOutput[] filtereds;
+        private final CloseResource[] closeThese;
+
+        private final OutputWorker[] outputWorkers;
+
+        private long pageCount;
+
+        public ScatterTransactionalPageOutput(ProcessState state, int taskIndex, int scatterCount)
+        {
+            this.state = state;
+            this.taskIndex = taskIndex;
+            this.scatterCount = scatterCount;
+
+            this.trans = new TransactionalPageOutput[scatterCount];
+            this.filtereds = new PageOutput[scatterCount];
+            this.closeThese = new CloseResource[scatterCount];
+            for (int i = 0; i < scatterCount; i++) {
+                closeThese[i] = new CloseResource();
+            }
+            this.outputWorkers = new OutputWorker[scatterCount];
+        }
+
+        public void openOutputs(OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource)
+        {
+            for (int i = 0; i < scatterCount; i++) {
+                int outputTaskIndex = taskIndex * scatterCount + i;
+                if (!state.getOutputTaskState(outputTaskIndex).isCommitted()) {
+                    TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
+                        outputPlugin.open(outputTaskSource, outputSchema, outputTaskIndex));
+                    trans[i] = tran;
+                    closeThese[i].closeThis(tran);
+                }
+            }
+        }
+
+        public void openFilters(List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources)
+        {
+            for (int i = 0; i < scatterCount; i++) {
+                TransactionalPageOutput tran = trans[i];
+                if (tran != null) {
+                    PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, trans[i]);
+                    filtereds[i] = filtered;
+                    closeThese[i].closeThis(filtered);
                 }
             }
-        });
+        }
+
+        public void startWorkers(ExecutorService outputExecutor)
+        {
+            for (int i = 0; i < scatterCount; i++) {
+                PageOutput filtered = filtereds[i];
+                if (filtered != null) {
+                    outputWorkers[i] = new OutputWorker(filtered, outputExecutor);
+                }
+            }
+        }
+
+        public void add(Page page)
+        {
+            OutputWorker worker = outputWorkers[(int) (pageCount % scatterCount)];
+            if (worker != null) {
+                try {
+                    worker.add(page);
+                }
+                catch (InterruptedException ex) {
+                    throw Throwables.propagate(ex);
+                }
+            }
+            pageCount++;
+        }
+
+        public void finish()
+        {
+            completeWorkers();
+            for (int i = 0; i < scatterCount; i++) {
+                if (trans[i] != null) {
+                    trans[i].finish();
+                }
+            }
+        }
+
+        public void close()
+        {
+            completeWorkers();
+            for (int i = 0; i < scatterCount; i++) {
+                closeThese[i].close();
+            }
+        }
+
+        public void abort()
+        {
+            completeWorkers();
+            for (int i = 0; i < scatterCount; i++) {
+                if (trans[i] != null) {
+                    trans[i].abort();
+                }
+            }
+        }
+
+        public TaskReport commit()
+        {
+            completeWorkers();
+            for (int i = 0; i < scatterCount; i++) {
+                if (trans[i] != null) {
+                    int outputTaskIndex = taskIndex * scatterCount + i;
+                    TaskReport outputTaskReport = trans[i].commit();
+                    trans[i] = null;  // don't abort
+                    if (outputTaskReport == null) {
+                        outputTaskReport = Exec.newTaskReport();
+                    }
+                    state.getOutputTaskState(outputTaskIndex).setTaskReport(outputTaskReport);
+                }
+            }
+            return null;
+        }
+
+        public void completeWorkers()
+        {
+            for (int i = 0; i < scatterCount; i++) {
+                OutputWorker worker = outputWorkers[i];
+                if (worker != null) {
+                    try {
+                        worker.add(DONE_PAGE);
+                    }
+                    catch (InterruptedException ex) {
+                        throw Throwables.propagate(ex);
+                    }
+                    Throwable error = null;
+                    try {
+                        error = worker.join();
+                    }
+                    catch (InterruptedException ex) {
+                        error = ex;
+                    }
+                    if (error != null) {
+                        throw Throwables.propagate(error);
+                    }
+                }
+            }
+        }
     }
 }