embulk 0.2.1 → 0.3.0

data/embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java

@@ -0,0 +1,18 @@
+package org.embulk.exec;
+
+public class PartialExecutionException
+        extends RuntimeException
+{
+    private final ResumeState resumeState;
+
+    public PartialExecutionException(Throwable cause, ResumeState resumeState)
+    {
+        super(cause);
+        this.resumeState = resumeState;
+    }
+
+    public ResumeState getResumeState()
+    {
+        return resumeState;
+    }
+}

data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java

@@ -0,0 +1,82 @@
+package org.embulk.exec;
+
+import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ExecSession;
+
+public class ResumeState
+{
+    private final ConfigSource execSessionConfigSource;
+    private final TaskSource inputTaskSource;
+    private final TaskSource outputTaskSource;
+    private final Schema inputSchema;
+    private final Schema outputSchema;
+    private final List<CommitReport> inputCommitReports;
+    private final List<CommitReport> outputCommitReports;
+
+    @JsonCreator
+    public ResumeState(
+            @JsonProperty("exec_task") ConfigSource execSessionConfigSource,
+            @JsonProperty("in_task") TaskSource inputTaskSource,
+            @JsonProperty("out_task") TaskSource outputTaskSource,
+            @JsonProperty("in_schema") Schema inputSchema,
+            @JsonProperty("out_schema") Schema outputSchema,
+            @JsonProperty("in_reports") List<CommitReport> inputCommitReports,
+            @JsonProperty("out_reports") List<CommitReport> outputCommitReports)
+    {
+        this.execSessionConfigSource = execSessionConfigSource;
+        this.inputTaskSource = inputTaskSource;
+        this.outputTaskSource = outputTaskSource;
+        this.inputSchema = inputSchema;
+        this.outputSchema = outputSchema;
+        this.inputCommitReports = inputCommitReports;
+        this.outputCommitReports = outputCommitReports;
+    }
+
+    @JsonProperty("exec_task")
+    public ConfigSource getExecSessionConfigSource()
+    {
+        return execSessionConfigSource;
+    }
+
+    @JsonProperty("in_task")
+    public TaskSource getInputTaskSource()
+    {
+        return inputTaskSource;
+    }
+
+    @JsonProperty("out_task")
+    public TaskSource getOutputTaskSource()
+    {
+        return outputTaskSource;
+    }
+
+    @JsonProperty("in_schema")
+    public Schema getInputSchema()
+    {
+        return inputSchema;
+    }
+
+    @JsonProperty("out_schema")
+    public Schema getOutputSchema()
+    {
+        return outputSchema;
+    }
+
+    @JsonProperty("in_reports")
+    public List<CommitReport> getInputCommitReports()
+    {
+        return inputCommitReports;
+    }
+
+    @JsonProperty("out_reports")
+    public List<CommitReport> getOutputCommitReports()
+    {
+        return outputCommitReports;
+    }
+}

data/embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java

@@ -12,7 +12,7 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.FormatterPlugin;
 import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.EncoderPlugin;
-//import org.embulk.spi.LineFilterPlugin;
+import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.GuessPlugin;
 
 public class JRubyPluginSource
@@ -50,8 +50,8 @@ public class JRubyPluginSource
             category = "decoder";
         } else if (EncoderPlugin.class.isAssignableFrom(iface)) {
             category = "encoder";
-        //} else if (LineFilterPlugin.class.isAssignableFrom(iface)) {
-        //    category = "line_filter";
+        } else if (FilterPlugin.class.isAssignableFrom(iface)) {
+            category = "filter";
         } else if (GuessPlugin.class.isAssignableFrom(iface)) {
             category = "guess";
         } else {

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -3,7 +3,11 @@ package org.embulk.spi;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.ILoggerFactory;
+import com.google.common.base.Optional;
 import com.google.inject.Injector;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
 import org.embulk.config.ModelManager;
 import org.embulk.config.CommitReport;
 import org.embulk.config.NextConfig;
@@ -26,18 +30,45 @@ public class ExecSession
     private final Timestamp transactionTime;
     private final DateTimeZone transactionTimeZone;
 
+    public interface SessionTask
+            extends Task
+    {
+        @Config("transaction_time")
+        @ConfigDefault("null")
+        Optional<Timestamp> getTransactionTime();
+
+        @Config("transaction_time_zone")
+        @ConfigDefault("\"UTC\"")
+        DateTimeZone getTransactionTimeZone();
+    }
+
     public ExecSession(Injector injector, ConfigSource execConfig)
     {
-        super();
+        this(injector, execConfig.loadConfig(SessionTask.class));
+    }
+
+    public ExecSession(Injector injector, TaskSource taskSource)
+    {
+        this(injector, taskSource.loadTask(SessionTask.class));
+    }
+
+    public ExecSession(Injector injector, SessionTask task)
+    {
         this.injector = injector;
         this.loggerFactory = injector.getInstance(ILoggerFactory.class);
         this.modelManager = injector.getInstance(ModelManager.class);
         this.pluginManager = injector.getInstance(PluginManager.class);
         this.bufferAllocator = injector.getInstance(BufferAllocator.class);
 
-        this.transactionTime = execConfig.get(Timestamp.class, "transaction_time",
-                Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
-        this.transactionTimeZone = execConfig.get(DateTimeZone.class, "transaction_time_zone", DateTimeZone.UTC);
+        this.transactionTime = task.getTransactionTime().or(Timestamp.ofEpochMilli(System.currentTimeMillis()));  // TODO get nanoseconds for default
+        this.transactionTimeZone = task.getTransactionTimeZone();
+    }
+
+    public ConfigSource getSessionConfigSource()
+    {
+        return newConfigSource()
+            .set("transaction_time", transactionTime)
+            .set("transaction_time_zone", transactionTimeZone);
     }
 
     public Injector getInjector()

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -10,10 +10,21 @@ public interface FileInputPlugin
 {
     public interface Control
     {
-        public List<CommitReport> run(TaskSource taskSource, int processorCount);
+        public List<CommitReport> run(TaskSource taskSource,
+                int processorCount);
     }
 
-    public NextConfig transaction(ConfigSource config, FileInputPlugin.Control control);
+    public NextConfig transaction(ConfigSource config,
+            FileInputPlugin.Control control);
 
-    public TransactionalFileInput open(TaskSource taskSource, int processorIndex);
+    public NextConfig resume(TaskSource taskSource,
+            int processorCount,
+            FileInputPlugin.Control control);
+
+    public void cleanup(TaskSource taskSource,
+            int processorCount,
+            List<CommitReport> successCommitReports);
+
+    public TransactionalFileInput open(TaskSource taskSource,
+            int processorIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -57,30 +57,61 @@ public class FileInputRunner
     public NextConfig transaction(ConfigSource config, final InputPlugin.Control control)
     {
         final RunnerTask task = config.loadConfig(RunnerTask.class);
-        final List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
-        final ParserPlugin parserPlugin = newParserPlugin(task);
-
-        return fileInputPlugin.transaction(config, new FileInputPlugin.Control() {
-            public List<CommitReport> run(final TaskSource fileInputTaskSource, final int processorCount)
-            {
-                final List<CommitReport> commitReports = new ArrayList<CommitReport>();
-                Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
-                    public void run(final List<TaskSource> decoderTaskSources)
-                    {
-                        parserPlugin.transaction(task.getParserConfig(), new ParserPlugin.Control() {
-                            public void run(final TaskSource parserTaskSource, final Schema schema)
-                            {
-                                task.setFileInputTaskSource(fileInputTaskSource);
-                                task.setDecoderTaskSources(decoderTaskSources);
-                                task.setParserTaskSource(parserTaskSource);
-                                commitReports.addAll(control.run(task.dump(), schema, processorCount));
-                            }
-                        });
-                    }
-                });
-                return commitReports;
-            }
-        });
+        return fileInputPlugin.transaction(config, new RunnerControl(task, control));
+    }
+
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            InputPlugin.Control control)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        return fileInputPlugin.resume(task.getFileInputTaskSource(), processorCount, new RunnerControl(task, control));
+    }
+
+    private class RunnerControl
+            implements FileInputPlugin.Control
+    {
+        private final RunnerTask task;
+        private final List<DecoderPlugin> decoderPlugins;
+        private final ParserPlugin parserPlugin;
+        private final InputPlugin.Control nextControl;
+
+        public RunnerControl(RunnerTask task, InputPlugin.Control nextControl)
+        {
+            this.task = task;
+            // create plugins earlier than run() to throw exceptions early
+            this.decoderPlugins = newDecoderPlugins(task);
+            this.parserPlugin = newParserPlugin(task);
+            this.nextControl = nextControl;
+        }
+
+        @Override
+        public List<CommitReport> run(final TaskSource fileInputTaskSource, final int processorCount)
+        {
+            final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+            Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
+                public void run(final List<TaskSource> decoderTaskSources)
+                {
+                    parserPlugin.transaction(task.getParserConfig(), new ParserPlugin.Control() {
+                        public void run(final TaskSource parserTaskSource, final Schema schema)
+                        {
+                            task.setFileInputTaskSource(fileInputTaskSource);
+                            task.setDecoderTaskSources(decoderTaskSources);
+                            task.setParserTaskSource(parserTaskSource);
+                            commitReports.addAll(nextControl.run(task.dump(), schema, processorCount));
+                        }
+                    });
+                }
+            });
+            return commitReports;
+        }
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    {
+        fileInputPlugin.cleanup(taskSource, processorCount, successCommitReports);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -16,5 +16,13 @@ public interface FileOutputPlugin
     public NextConfig transaction(ConfigSource config, int processorCount,
             FileOutputPlugin.Control control);
 
+    public NextConfig resume(TaskSource taskSource,
+            int processorCount,
+            FileOutputPlugin.Control control);
+
+    public void cleanup(TaskSource taskSource,
+            int processorCount,
+            List<CommitReport> successCommitReports);
+
     public TransactionalFileOutput open(TaskSource taskSource, int processorIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -60,30 +60,63 @@ public class FileOutputRunner
             final OutputPlugin.Control control)
     {
         final RunnerTask task = config.loadConfig(RunnerTask.class);
-        final List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
-        final FormatterPlugin formatterPlugin = newFormatterPlugin(task);
-
-        return fileOutputPlugin.transaction(config, processorCount, new FileOutputPlugin.Control() {
-            public List<CommitReport> run(final TaskSource fileOutputTaskSource)
-            {
-                final List<CommitReport> commitReports = new ArrayList<CommitReport>();
-                Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
-                    public void run(final List<TaskSource> encoderTaskSources)
-                    {
-                        formatterPlugin.transaction(task.getFormatterConfig(), schema, new FormatterPlugin.Control() {
-                            public void run(final TaskSource formatterTaskSource)
-                            {
-                                task.setFileOutputTaskSource(fileOutputTaskSource);
-                                task.setEncoderTaskSources(encoderTaskSources);
-                                task.setFormatterTaskSource(formatterTaskSource);
-                                commitReports.addAll(control.run(task.dump()));
-                            }
-                        });
-                    }
-                });
-                return commitReports;
-            }
-        });
+        return fileOutputPlugin.transaction(config, processorCount, new RunnerControl(schema, task, control));
+    }
+
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            final OutputPlugin.Control control)
+    {
+        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
+        return fileOutputPlugin.resume(task.getFileOutputTaskSource(), processorCount, new RunnerControl(schema, task, control));
+    }
+
+    private class RunnerControl
+            implements FileOutputPlugin.Control
+    {
+        private final Schema schema;
+        private final RunnerTask task;
+        private final List<EncoderPlugin> encoderPlugins;
+        private final FormatterPlugin formatterPlugin;
+        private final OutputPlugin.Control nextControl;
+
+        public RunnerControl(Schema schema, RunnerTask task, OutputPlugin.Control nextControl)
+        {
+            this.schema = schema;
+            this.task = task;
+            // create plugins earlier than run() to throw exceptions early
+            this.encoderPlugins = newEncoderPlugins(task);
+            this.formatterPlugin = newFormatterPlugin(task);
+            this.nextControl = nextControl;
+        }
+
+        @Override
+        public List<CommitReport> run(final TaskSource fileOutputTaskSource)
+        {
+            final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+            Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
+                public void run(final List<TaskSource> encoderTaskSources)
+                {
+                    formatterPlugin.transaction(task.getFormatterConfig(), schema, new FormatterPlugin.Control() {
+                        public void run(final TaskSource formatterTaskSource)
+                        {
+                            task.setFileOutputTaskSource(fileOutputTaskSource);
+                            task.setEncoderTaskSources(encoderTaskSources);
+                            task.setFormatterTaskSource(formatterTaskSource);
+                            commitReports.addAll(nextControl.run(task.dump()));
+                        }
+                    });
+                }
+            });
+            return commitReports;
+        }
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    {
+        fileOutputPlugin.cleanup(taskSource, processorCount, successCommitReports);
     }
 
     @Override

data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java

@@ -0,0 +1,21 @@
+package org.embulk.spi;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.NextConfig;
+import org.embulk.config.CommitReport;
+
+public interface FilterPlugin
+{
+    public interface Control
+    {
+        public void run(TaskSource taskSource, Schema outputSchema);
+    }
+
+    public void transaction(ConfigSource config, Schema inputSchema,
+            FilterPlugin.Control control);
+
+    public PageOutput open(TaskSource taskSource, Schema inputSchema,
+            Schema outputSchema, PageOutput output);
+}

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -10,11 +10,22 @@ public interface InputPlugin
 {
     public interface Control
     {
-        public List<CommitReport> run(TaskSource taskSource, Schema schema, int processorCount);
+        public List<CommitReport> run(TaskSource taskSource,
+                Schema schema, int processorCount);
     }
 
-    public NextConfig transaction(ConfigSource config, InputPlugin.Control control);
+    public NextConfig transaction(ConfigSource config,
+            InputPlugin.Control control);
 
-    public CommitReport run(TaskSource taskSource, Schema schema, int processorIndex,
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            InputPlugin.Control control);
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports);
+
+    public CommitReport run(TaskSource taskSource,
+            Schema schema, int processorIndex,
             PageOutput output);
 }