embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/FileInput.java

@@ -0,0 +1,11 @@
+package org.embulk.spi;
+
+public interface FileInput
+        extends AutoCloseable
+{
+    public boolean nextFile();
+
+    public Buffer poll();
+
+    public void close();
+}

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -0,0 +1,19 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+
+public interface FileInputPlugin
+{
+    public interface Control
+    {
+        public List<CommitReport> run(TaskSource taskSource, int processorCount);
+    }
+
+    public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control);
+
+    public TransactionalFileInput open(TaskSource taskSource, int processorIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -0,0 +1,113 @@
+package org.embulk.spi;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.util.Decoders;
+
+public class FileInputRunner
+        implements InputPlugin
+{
+    private final FileInputPlugin fileInputPlugin;
+
+    public FileInputRunner(FileInputPlugin fileInputPlugin)
+    {
+        this.fileInputPlugin = fileInputPlugin;
+    }
+
+    private interface RunnerTask extends Task
+    {
+        // TODO "type" needed?
+
+        @Config("decoders")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getDecoderConfigs();
+
+        @Config("parser")
+        public ConfigSource getParserConfig();
+
+        public void setFileInputTaskSource(TaskSource v);
+        public TaskSource getFileInputTaskSource();
+
+        public void setDecoderTaskSources(List<TaskSource> v);
+        public List<TaskSource> getDecoderTaskSources();
+
+        public void setParserTaskSource(TaskSource v);
+        public TaskSource getParserTaskSource();
+    }
+
+    protected List<DecoderPlugin> newDecoderPlugins(RunnerTask task)
+    {
+        return Decoders.newDecoderPlugins(Exec.session(), task.getDecoderConfigs());
+    }
+
+    protected ParserPlugin newParserPlugin(RunnerTask task)
+    {
+        return Exec.newPlugin(ParserPlugin.class, task.getParserConfig().get(PluginType.class, "type"));
+    }
+
+    @Override
+    public NextConfig transaction(ConfigSource config, final InputPlugin.Control control)
+    {
+        final RunnerTask task = config.loadConfig(RunnerTask.class);
+        final List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
+        final ParserPlugin parserPlugin = newParserPlugin(task);
+
+        return fileInputPlugin.transaction(config, new FileInputPlugin.Control() {
+            public List<CommitReport> run(final TaskSource fileInputTaskSource, final int processorCount)
+            {
+                final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+                Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
+                    public void run(final List<TaskSource> decoderTaskSources)
+                    {
+                        parserPlugin.transaction(task.getParserConfig(), new ParserPlugin.Control() {
+                            public void run(final TaskSource parserTaskSource, final Schema schema)
+                            {
+                                task.setFileInputTaskSource(fileInputTaskSource);
+                                task.setDecoderTaskSources(decoderTaskSources);
+                                task.setParserTaskSource(parserTaskSource);
+                                commitReports.addAll(control.run(task.dump(), schema, processorCount));
+                            }
+                        });
+                    }
+                });
+                return commitReports;
+            }
+        });
+    }
+
+    @Override
+    public CommitReport run(TaskSource taskSource, Schema schema, int processorIndex,
+            PageOutput output)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
+        ParserPlugin parserPlugin = newParserPlugin(task);
+
+        TransactionalFileInput tran = fileInputPlugin.open(task.getFileInputTaskSource(), processorIndex);
+        FileInput fileInput = tran;
+        try {
+            fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), fileInput);
+            parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
+
+            CommitReport report = tran.commit();  // TODO check output.finish() is called. wrap
+            tran = null;
+            return report;
+        } finally {
+            try {
+                if (tran != null) {
+                    tran.abort();
+                }
+            } finally {
+                fileInput.close();
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java

@@ -0,0 +1,13 @@
+package org.embulk.spi;
+
+public interface FileOutput
+        extends AutoCloseable
+{
+    public void nextFile();
+
+    public void add(Buffer buffer);
+
+    public void finish();
+
+    public void close();
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -0,0 +1,20 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+
+public interface FileOutputPlugin
+{
+    public interface Control
+    {
+        public List<CommitReport> run(TaskSource taskSource);
+    }
+
+    public NextConfig transaction(ConfigSource config, int processorCount,
+            FileOutputPlugin.Control control);
+
+    public TransactionalFileOutput open(TaskSource taskSource, int processorIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -0,0 +1,167 @@
+package org.embulk.spi;
+
+import java.util.List;
+import java.util.ArrayList;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.util.Encoders;
+
+public class FileOutputRunner
+        implements OutputPlugin
+{
+    private final FileOutputPlugin fileOutputPlugin;
+
+    public FileOutputRunner(FileOutputPlugin fileOutputPlugin)
+    {
+        this.fileOutputPlugin = fileOutputPlugin;
+    }
+
+    private interface RunnerTask extends Task
+    {
+        @Config("type")
+        public PluginType getType();
+
+        @Config("encoders")
+        @ConfigDefault("[]")
+        public List<ConfigSource> getEncoderConfigs();
+
+        @Config("formatter")
+        public ConfigSource getFormatterConfig();
+
+        public void setFileOutputTaskSource(TaskSource v);
+        public TaskSource getFileOutputTaskSource();
+
+        public void setEncoderTaskSources(List<TaskSource> v);
+        public List<TaskSource> getEncoderTaskSources();
+
+        public void setFormatterTaskSource(TaskSource v);
+        public TaskSource getFormatterTaskSource();
+    }
+
+    protected List<EncoderPlugin> newEncoderPlugins(RunnerTask task)
+    {
+        return Encoders.newEncoderPlugins(Exec.session(), task.getEncoderConfigs());
+    }
+
+    protected FormatterPlugin newFormatterPlugin(RunnerTask task)
+    {
+        return Exec.newPlugin(FormatterPlugin.class, task.getFormatterConfig().get(PluginType.class, "type"));
+    }
+
+    @Override
+    public NextConfig transaction(ConfigSource config,
+            final Schema schema, final int processorCount,
+            final OutputPlugin.Control control)
+    {
+        final RunnerTask task = config.loadConfig(RunnerTask.class);
+        final List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
+        final FormatterPlugin formatterPlugin = newFormatterPlugin(task);
+
+        return fileOutputPlugin.transaction(config, processorCount, new FileOutputPlugin.Control() {
+            public List<CommitReport> run(final TaskSource fileOutputTaskSource)
+            {
+                final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+                Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
+                    public void run(final List<TaskSource> encoderTaskSources)
+                    {
+                        formatterPlugin.transaction(task.getFormatterConfig(), schema, new FormatterPlugin.Control() {
+                            public void run(final TaskSource formatterTaskSource)
+                            {
+                                task.setFileOutputTaskSource(fileOutputTaskSource);
+                                task.setEncoderTaskSources(encoderTaskSources);
+                                task.setFormatterTaskSource(formatterTaskSource);
+                                commitReports.addAll(control.run(task.dump()));
+                            }
+                        });
+                    }
+                });
+                return commitReports;
+            }
+        });
+    }
+
+    @Override
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
+        FormatterPlugin formatterPlugin = newFormatterPlugin(task);
+
+        TransactionalFileOutput tran = null;
+        FileOutput fileOutput = null;
+        PageOutput output = null;
+        try {
+            fileOutput = tran = fileOutputPlugin.open(task.getFileOutputTaskSource(), processorIndex);
+
+            fileOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), fileOutput);
+            output = formatterPlugin.open(task.getFormatterTaskSource(), schema, fileOutput);
+            fileOutput = null;
+
+            TransactionalPageOutput ret = new DelegateTransactionalPageOutput(tran, output);
+            tran = null;
+            output = null;
+            return ret;
+
+        } finally {
+            if (output != null) {
+                output.close();
+            }
+            if (fileOutput != null) {
+                fileOutput.close();
+            }
+            if (tran != null) {
+                tran.abort();
+            }
+        }
+    }
+
+    private static class DelegateTransactionalPageOutput
+            implements TransactionalPageOutput
+    {
+        private final Transactional tran;
+        private final PageOutput output;
+
+        public DelegateTransactionalPageOutput(Transactional tran, PageOutput output)
+        {
+            this.tran = tran;
+            this.output = output;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            output.add(page);
+        }
+
+        @Override
+        public void finish()
+        {
+            output.finish();
+        }
+
+        @Override
+        public void close()
+        {
+            output.close();
+        }
+
+        @Override
+        public void abort()
+        {
+            tran.abort();
+        }
+
+        @Override
+        public CommitReport commit()
+        {
+            // TODO check finished
+            return tran.commit();
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java

@@ -0,0 +1,18 @@
+package org.embulk.spi;
+
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+
+public interface FormatterPlugin
+{
+    public interface Control
+    {
+        public void run(TaskSource taskSource);
+    }
+
+    public void transaction(ConfigSource config, Schema schema,
+            FormatterPlugin.Control control);
+
+    public PageOutput open(TaskSource taskSource, Schema schema,
+            FileOutput output);
+}

data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java

@@ -0,0 +1,9 @@
+package org.embulk.spi;
+
+import org.embulk.config.NextConfig;
+import org.embulk.config.ConfigSource;
+
+public interface GuessPlugin
+{
+    public NextConfig guess(ConfigSource config, Buffer sample);
+}

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -0,0 +1,20 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+
+public interface InputPlugin
+{
+    public interface Control
+    {
+        public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount);
+    }
+
+    public NextConfig transaction(ConfigSource config, InputPlugin.Control control);
+
+    public CommitReport run(TaskSource taskSource, Schema schema, int processorIndex,
+            PageOutput output);
+}

data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java

@@ -0,0 +1,21 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+
+public interface OutputPlugin
+{
+    public interface Control
+    {
+        public List<CommitReport> run(TaskSource taskSource);
+    }
+
+    public NextConfig transaction(ConfigSource config,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control);
+
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/Page.java

@@ -0,0 +1,45 @@
+package org.embulk.spi;
+
+import java.util.List;
+
+public class Page
+{
+    private final Buffer buffer;
+    private List<String> stringReferences;
+
+    protected Page(Buffer buffer)
+    {
+        this.buffer = buffer;
+    }
+
+    public static Page allocate(int length)
+    {
+        return new Page(Buffer.allocate(length));
+    }
+
+    public static Page wrap(Buffer buffer)
+    {
+        return new Page(buffer);
+    }
+
+    public Page setStringReferences(List<String> values)
+    {
+        this.stringReferences = values;
+        return this;
+    }
+
+    public String getStringReference(int index)
+    {
+        return stringReferences.get(index);
+    }
+
+    public void release()
+    {
+        buffer.release();
+    }
+
+    public Buffer buffer()
+    {
+        return buffer;
+    }
+}