embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java

@@ -0,0 +1,19 @@
+package org.embulk.spi;
+
+import org.embulk.config.ConfigSource;
+
+public interface ExecutorPlugin
+{
+    interface Executor
+    {
+        void execute(ProcessTask task, ProcessState state);
+    }
+
+    interface Control
+    {
+        void transaction(Schema executorSchema, int outputTaskCount, Executor executor);
+    }
+
+    void transaction(ConfigSource config, Schema outputSchema, int inputTaskCount,
+            ExecutorPlugin.Control control);
+}

data/embulk-core/src/main/java/org/embulk/spi/Extension.java

@@ -0,0 +1,44 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.google.inject.Module;
+import org.embulk.config.ConfigSource;
+
+/**
+ * Extension is a module to extend the execution framework using Guice.
+ * Unlike plugins, extensions can overwrite or add core components such as
+ * BufferManager, PluginSource, etc.
+ * Extension is not designed for users but for framework developers to make
+ * core components loosely coupled.
+ *
+ * An example extension to add a custom PluginSource will be as following:
+ *
+ * <code>
+ * class MyPluginSourceExtension
+ *         implements Extension, Module
+ * {
+ *     public static class MyPluginSource
+ *             implements PluginSource
+ *     {
+ *         // ...
+ *     }
+ *
+ *     {@literal @}Override
+ *     public void configure(Binder binder)
+ *     {
+ *         Multibinder&lt;PluginSource&gt; multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
+ *         multibinder.addBinding().to(MyPluginSource.class);
+ *     }
+ *
+ *     {@literal @}Override
+ *     public List&lt;Module&gt; getModules()
+ *     {
+ *         return ImmutableList.&lt;Module&gt;of(this);
+ *     }
+ * }
+ * </code>
+ */
+public interface Extension
+{
+    List<Module> getModules(ConfigSource systemConfig);
+}

data/embulk-core/src/main/java/org/embulk/spi/FileInput.java

@@ -0,0 +1,11 @@
+package org.embulk.spi;
+
+public interface FileInput
+        extends AutoCloseable
+{
+    boolean nextFile();
+
+    Buffer poll();
+
+    void close();
+}

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -0,0 +1,30 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+
+public interface FileInputPlugin
+{
+    interface Control
+    {
+        List<TaskReport> run(TaskSource taskSource,
+                int taskCount);
+    }
+
+    ConfigDiff transaction(ConfigSource config,
+            FileInputPlugin.Control control);
+
+    ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control);
+
+    void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports);
+
+    TransactionalFileInput open(TaskSource taskSource,
+            int taskIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -0,0 +1,162 @@
+package org.embulk.spi;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.compat.PluginWrappers;
+import org.embulk.spi.util.Decoders;
+import org.embulk.exec.GuessExecutor;
+import org.embulk.exec.SamplingParserPlugin;
+import org.embulk.exec.NoSampleException;
+
+public class FileInputRunner
+        implements InputPlugin
+{
+    private final FileInputPlugin fileInputPlugin;
+
+    public FileInputRunner(FileInputPlugin fileInputPlugin)
+    {
+        this.fileInputPlugin = fileInputPlugin;
+    }
+
+    private interface RunnerTask extends Task
+    {
+        // TODO "type" needed?
+
+        @Config("decoders")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getDecoderConfigs();
+
+        @Config("parser")
+        public ConfigSource getParserConfig();
+
+        public void setFileInputTaskSource(TaskSource v);
+        public TaskSource getFileInputTaskSource();
+
+        public void setDecoderTaskSources(List<TaskSource> v);
+        public List<TaskSource> getDecoderTaskSources();
+
+        public void setParserTaskSource(TaskSource v);
+        public TaskSource getParserTaskSource();
+    }
+
+    protected List<DecoderPlugin> newDecoderPlugins(RunnerTask task)
+    {
+        return Decoders.newDecoderPlugins(Exec.session(), task.getDecoderConfigs());
+    }
+
+    protected ParserPlugin newParserPlugin(RunnerTask task)
+    {
+        return Exec.newPlugin(ParserPlugin.class, task.getParserConfig().get(PluginType.class, "type"));
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, final InputPlugin.Control control)
+    {
+        final RunnerTask task = config.loadConfig(RunnerTask.class);
+        return fileInputPlugin.transaction(config, new RunnerControl(task, control));
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        return fileInputPlugin.resume(task.getFileInputTaskSource(), taskCount, new RunnerControl(task, control));
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        Buffer sample = SamplingParserPlugin.runFileInputSampling(this, config);
+        if (sample.limit() == 0) {
+            throw new NoSampleException("Can't get sample data because the first input file is empty");
+        }
+
+        GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
+        return guessExecutor.guessParserConfig(sample, config, Exec.session().getSessionExecConfig());
+    }
+
+    private class RunnerControl
+            implements FileInputPlugin.Control
+    {
+        private final RunnerTask task;
+        private final List<DecoderPlugin> decoderPlugins;
+        private final ParserPlugin parserPlugin;
+        private final InputPlugin.Control nextControl;
+
+        public RunnerControl(RunnerTask task, InputPlugin.Control nextControl)
+        {
+            this.task = task;
+            // create plugins earlier than run() to throw exceptions early
+            this.decoderPlugins = newDecoderPlugins(task);
+            this.parserPlugin = newParserPlugin(task);
+            this.nextControl = nextControl;
+        }
+
+        @Override
+        public List<TaskReport> run(final TaskSource fileInputTaskSource, final int taskCount)
+        {
+            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
+            Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
+                public void run(final List<TaskSource> decoderTaskSources)
+                {
+                    parserPlugin.transaction(task.getParserConfig(), new ParserPlugin.Control() {
+                        public void run(final TaskSource parserTaskSource, final Schema schema)
+                        {
+                            task.setFileInputTaskSource(fileInputTaskSource);
+                            task.setDecoderTaskSources(decoderTaskSources);
+                            task.setParserTaskSource(parserTaskSource);
+                            taskReports.addAll(nextControl.run(task.dump(), schema, taskCount));
+                        }
+                    });
+                }
+            });
+            return taskReports;
+        }
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+        fileInputPlugin.cleanup(taskSource, taskCount, successTaskReports);
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
+        ParserPlugin parserPlugin = newParserPlugin(task);
+
+        TransactionalFileInput tran = PluginWrappers.transactionalFileInput(
+                fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex));
+        FileInput fileInput = tran;
+        try {
+            fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), fileInput);
+            parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
+
+            TaskReport report = tran.commit();  // TODO check output.finish() is called. wrap
+            tran = null;
+            return report;
+        } finally {
+            try {
+                if (tran != null) {
+                    tran.abort();
+                }
+            } finally {
+                fileInput.close();
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.spi;
+
+public interface FileOutput
+        extends AutoCloseable
+{
+    void nextFile();
+
+    void add(Buffer buffer);
+
+    void finish();
+
+    void close();
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -0,0 +1,28 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+
+public interface FileOutputPlugin
+{
+    interface Control
+    {
+        List<TaskReport> run(TaskSource taskSource);
+    }
+
+    ConfigDiff transaction(ConfigSource config, int taskCount,
+            FileOutputPlugin.Control control);
+
+    ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileOutputPlugin.Control control);
+
+    void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<TaskReport> successTaskReports);
+
+    TransactionalFileOutput open(TaskSource taskSource, int taskIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -0,0 +1,202 @@
+package org.embulk.spi;
+
+import java.util.List;
+import java.util.ArrayList;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.plugin.PluginType;
+import org.embulk.plugin.compat.PluginWrappers;
+import org.embulk.spi.util.Encoders;
+
+public class FileOutputRunner
+        implements OutputPlugin
+{
+    private final FileOutputPlugin fileOutputPlugin;
+
+    public FileOutputRunner(FileOutputPlugin fileOutputPlugin)
+    {
+        this.fileOutputPlugin = fileOutputPlugin;
+    }
+
+    private interface RunnerTask extends Task
+    {
+        @Config("type")
+        public PluginType getType();
+
+        @Config("encoders")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getEncoderConfigs();
+
+        @Config("formatter")
+        public ConfigSource getFormatterConfig();
+
+        public void setFileOutputTaskSource(TaskSource v);
+        public TaskSource getFileOutputTaskSource();
+
+        public void setEncoderTaskSources(List<TaskSource> v);
+        public List<TaskSource> getEncoderTaskSources();
+
+        public void setFormatterTaskSource(TaskSource v);
+        public TaskSource getFormatterTaskSource();
+    }
+
+    protected List<EncoderPlugin> newEncoderPlugins(RunnerTask task)
+    {
+        return Encoders.newEncoderPlugins(Exec.session(), task.getEncoderConfigs());
+    }
+
+    protected FormatterPlugin newFormatterPlugin(RunnerTask task)
+    {
+        return Exec.newPlugin(FormatterPlugin.class, task.getFormatterConfig().get(PluginType.class, "type"));
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            final Schema schema, final int taskCount,
+            final OutputPlugin.Control control)
+    {
+        final RunnerTask task = config.loadConfig(RunnerTask.class);
+        return fileOutputPlugin.transaction(config, taskCount, new RunnerControl(schema, task, control));
+    }
+
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            final OutputPlugin.Control control)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        return fileOutputPlugin.resume(task.getFileOutputTaskSource(), taskCount, new RunnerControl(schema, task, control));
+    }
+
+    private class RunnerControl
+            implements FileOutputPlugin.Control
+    {
+        private final Schema schema;
+        private final RunnerTask task;
+        private final List<EncoderPlugin> encoderPlugins;
+        private final FormatterPlugin formatterPlugin;
+        private final OutputPlugin.Control nextControl;
+
+        public RunnerControl(Schema schema, RunnerTask task, OutputPlugin.Control nextControl)
+        {
+            this.schema = schema;
+            this.task = task;
+            // create plugins earlier than run() to throw exceptions early
+            this.encoderPlugins = newEncoderPlugins(task);
+            this.formatterPlugin = newFormatterPlugin(task);
+            this.nextControl = nextControl;
+        }
+
+        @Override
+        public List<TaskReport> run(final TaskSource fileOutputTaskSource)
+        {
+            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
+            Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
+                public void run(final List<TaskSource> encoderTaskSources)
+                {
+                    formatterPlugin.transaction(task.getFormatterConfig(), schema, new FormatterPlugin.Control() {
+                        public void run(final TaskSource formatterTaskSource)
+                        {
+                            task.setFileOutputTaskSource(fileOutputTaskSource);
+                            task.setEncoderTaskSources(encoderTaskSources);
+                            task.setFormatterTaskSource(formatterTaskSource);
+                            taskReports.addAll(nextControl.run(task.dump()));
+                        }
+                    });
+                }
+            });
+            return taskReports;
+        }
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successtaskReports)
+    {
+        fileOutputPlugin.cleanup(taskSource, taskCount, successtaskReports);
+    }
+
+    @Override
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
+        FormatterPlugin formatterPlugin = newFormatterPlugin(task);
+
+        TransactionalFileOutput tran = null;
+        FileOutput fileOutput = null;
+        PageOutput output = null;
+        try {
+            fileOutput = tran = PluginWrappers.transactionalFileOutput(
+                    fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex));
+
+            fileOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), fileOutput);
+            output = formatterPlugin.open(task.getFormatterTaskSource(), schema, fileOutput);
+            fileOutput = null;
+
+            TransactionalPageOutput ret = new DelegateTransactionalPageOutput(tran, output);
+            tran = null;
+            output = null;
+            return ret;
+
+        } finally {
+            if (output != null) {
+                output.close();
+            }
+            if (fileOutput != null) {
+                fileOutput.close();
+            }
+            if (tran != null) {
+                tran.abort();
+            }
+        }
+    }
+
+    private static class DelegateTransactionalPageOutput
+            implements TransactionalPageOutput
+    {
+        private final Transactional tran;
+        private final PageOutput output;
+
+        public DelegateTransactionalPageOutput(Transactional tran, PageOutput output)
+        {
+            this.tran = tran;
+            this.output = output;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            output.add(page);
+        }
+
+        @Override
+        public void finish()
+        {
+            output.finish();
+        }
+
+        @Override
+        public void close()
+        {
+            output.close();
+        }
+
+        @Override
+        public void abort()
+        {
+            tran.abort();
+        }
+
+        @Override
+        public TaskReport commit()
+        {
+            // TODO check finished
+            return tran.commit();
+        }
+    }
+}