embulk 0.6.27 → 0.7.0

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -4,13 +4,13 @@ import java.util.List;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface FileOutputPlugin
 {
     interface Control
     {
-        List<CommitReport> run(TaskSource taskSource);
+        List<TaskReport> run(TaskSource taskSource);
     }
 
     ConfigDiff transaction(ConfigSource config, int taskCount,
@@ -22,7 +22,7 @@ public interface FileOutputPlugin
 
     void cleanup(TaskSource taskSource,
             int taskCount,
-            List<CommitReport> successCommitReports);
+            List<TaskReport> successTaskReports);
 
     TransactionalFileOutput open(TaskSource taskSource, int taskIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -6,10 +6,11 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.plugin.PluginType;
+import org.embulk.plugin.compat.PluginWrappers;
 import org.embulk.spi.util.Encoders;
 
 public class FileOutputRunner
@@ -91,9 +92,9 @@ public class FileOutputRunner
         }
 
         @Override
-        public List<CommitReport> run(final TaskSource fileOutputTaskSource)
+        public List<TaskReport> run(final TaskSource fileOutputTaskSource)
         {
-            final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
             Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
                 public void run(final List<TaskSource> encoderTaskSources)
                 {
@@ -103,20 +104,20 @@ public class FileOutputRunner
                             task.setFileOutputTaskSource(fileOutputTaskSource);
                             task.setEncoderTaskSources(encoderTaskSources);
                             task.setFormatterTaskSource(formatterTaskSource);
-                            commitReports.addAll(nextControl.run(task.dump()));
+                            taskReports.addAll(nextControl.run(task.dump()));
                         }
                     });
                 }
             });
-            return commitReports;
+            return taskReports;
         }
     }
 
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successtaskReports)
     {
-        fileOutputPlugin.cleanup(taskSource, taskCount, successCommitReports);
+        fileOutputPlugin.cleanup(taskSource, taskCount, successtaskReports);
     }
 
     @Override
@@ -130,7 +131,8 @@ public class FileOutputRunner
         FileOutput fileOutput = null;
         PageOutput output = null;
         try {
-            fileOutput = tran = fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex);
+            fileOutput = tran = PluginWrappers.transactionalFileOutput(
+                    fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex));
 
             fileOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), fileOutput);
             output = formatterPlugin.open(task.getFormatterTaskSource(), schema, fileOutput);
@@ -191,7 +193,7 @@ public class FileOutputRunner
         }
 
         @Override
-        public CommitReport commit()
+        public TaskReport commit()
         {
             // TODO check finished
             return tran.commit();

data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java

@@ -1,10 +1,7 @@
 package org.embulk.spi;
 
-import java.util.List;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
 
 public interface FilterPlugin
 {

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -4,13 +4,13 @@ import java.util.List;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface InputPlugin
 {
     interface Control
     {
-        List<CommitReport> run(TaskSource taskSource,
+        List<TaskReport> run(TaskSource taskSource,
                 Schema schema, int taskCount);
     }
 
@@ -23,9 +23,9 @@ public interface InputPlugin
 
     void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports);
+            List<TaskReport> successTaskReports);
 
-    CommitReport run(TaskSource taskSource,
+    TaskReport run(TaskSource taskSource,
             Schema schema, int taskIndex,
             PageOutput output);
 

data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java

@@ -4,13 +4,13 @@ import java.util.List;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface OutputPlugin
 {
     interface Control
     {
-        List<CommitReport> run(TaskSource taskSource);
+        List<TaskReport> run(TaskSource taskSource);
     }
 
     ConfigDiff transaction(ConfigSource config,
@@ -23,7 +23,7 @@ public interface OutputPlugin
 
     void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports);
+            List<TaskReport> successTaskReports);
 
     TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/PageReader.java

@@ -1,6 +1,5 @@
 package org.embulk.spi;
 
-import java.util.Iterator;
 import io.airlift.slice.Slice;
 import io.airlift.slice.Slices;
 import org.embulk.spi.time.Timestamp;

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -1,17 +1,12 @@
 package org.embulk.spi;
 
 import java.util.List;
-import com.google.common.base.Function;
-import com.google.common.collect.Lists;
-import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.embulk.plugin.PluginType;
 import org.embulk.config.TaskSource;
-import org.embulk.spi.Schema;
 import org.embulk.spi.util.Executors;
-import org.embulk.spi.type.TimestampType;
 
 public class ProcessTask
 {

data/embulk-core/src/main/java/org/embulk/spi/SchemaConfig.java

@@ -2,7 +2,6 @@ package org.embulk.spi;
 
 import java.util.List;
 import java.util.Objects;
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;

data/embulk-core/src/main/java/org/embulk/spi/TaskState.java

@@ -1,13 +1,14 @@
 package org.embulk.spi;
 
 import com.google.common.base.Optional;
+import org.embulk.config.TaskReport;
 import org.embulk.config.CommitReport;
 
 public class TaskState
 {
     private volatile boolean started = false;
     private volatile boolean finished = false;
-    private volatile Optional<CommitReport> commitReport = Optional.absent();
+    private volatile Optional<TaskReport> taskReport = Optional.absent();
     private volatile Optional<Throwable> exception = Optional.absent();
 
     public void start()
@@ -21,10 +22,17 @@ public class TaskState
         this.finished = true;
     }
 
+    public void setTaskReport(TaskReport taskReport)
+    {
+        this.started = true;
+        this.taskReport = Optional.of(taskReport);
+    }
+
+    @Deprecated
     public void setCommitReport(CommitReport commitReport)
     {
         this.started = true;
-        this.commitReport = Optional.of(commitReport);
+        this.taskReport = Optional.<TaskReport>of(commitReport);
     }
 
     public void setException(Throwable exception)
@@ -51,12 +59,19 @@ public class TaskState
 
     public boolean isCommitted()
     {
-        return commitReport.isPresent();
+        return taskReport.isPresent();
+    }
+
+    public Optional<TaskReport> getTaskReport()
+    {
+        return taskReport;
     }
 
+    @Deprecated
+    @SuppressWarnings("unchecked")
     public Optional<CommitReport> getCommitReport()
     {
-        return commitReport;
+        return (Optional) taskReport;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport;
     }
 
     public Optional<Throwable> getException()

data/embulk-core/src/main/java/org/embulk/spi/TempFileSpace.java

@@ -1,8 +1,5 @@
 package org.embulk.spi;
 
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;

data/embulk-core/src/main/java/org/embulk/spi/Transactional.java

@@ -1,10 +1,10 @@
 package org.embulk.spi;
 
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface Transactional
 {
     void abort();
 
-    CommitReport commit();
+    TaskReport commit();
 }

data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileInput.java

@@ -1,6 +1,6 @@
 package org.embulk.spi;
 
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface TransactionalFileInput
         extends Transactional, FileInput
@@ -13,5 +13,5 @@ public interface TransactionalFileInput
 
     void abort();
 
-    CommitReport commit();
+    TaskReport commit();
 }

data/embulk-core/src/main/java/org/embulk/spi/TransactionalFileOutput.java

@@ -1,6 +1,6 @@
 package org.embulk.spi;
 
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface TransactionalFileOutput
         extends Transactional, FileOutput
@@ -15,5 +15,5 @@ public interface TransactionalFileOutput
 
     void abort();
 
-    CommitReport commit();
+    TaskReport commit();
 }

data/embulk-core/src/main/java/org/embulk/spi/TransactionalPageOutput.java

@@ -1,6 +1,6 @@
 package org.embulk.spi;
 
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface TransactionalPageOutput
         extends Transactional, PageOutput
@@ -13,5 +13,5 @@ public interface TransactionalPageOutput
 
     void abort();
 
-    CommitReport commit();
+    TaskReport commit();
 }

data/embulk-core/src/main/java/org/embulk/spi/time/DateTimeZoneSerDe.java

@@ -3,7 +3,6 @@ package org.embulk.spi.time;
 import java.io.IOException;
 import org.joda.time.DateTimeZone;
 import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.DeserializationContext;
@@ -11,7 +10,6 @@ import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
 import com.fasterxml.jackson.module.guice.ObjectMapperModule;
-import org.embulk.config.ModelManager;
 
 public class DateTimeZoneSerDe
 {

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormat.java

@@ -1,9 +1,6 @@
 package org.embulk.spi.time;
 
 import java.util.Set;
-import java.util.Date;
-import java.text.SimpleDateFormat;
-import java.text.ParsePosition;
 import com.google.common.collect.ImmutableSet;
 import com.fasterxml.jackson.annotation.JsonValue;
 import com.fasterxml.jackson.annotation.JsonCreator;

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampFormatter.java

@@ -9,7 +9,6 @@ import org.jruby.util.RubyDateFormat;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigException;
 import org.embulk.spi.util.LineEncoder;
 
 public class TimestampFormatter

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampParser.java

@@ -6,7 +6,6 @@ import org.jruby.embed.ScriptingContainer;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigException;
 import static org.embulk.spi.time.TimestampFormat.parseDateTimeZone;
 
 public class TimestampParser

data/embulk-core/src/main/java/org/embulk/spi/time/TimestampSerDe.java

@@ -2,7 +2,6 @@ package org.embulk.spi.time;
 
 import java.io.IOException;
 import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.databind.Module;
 import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.DeserializationContext;

data/embulk-core/src/main/java/org/embulk/spi/type/TypeDeserializer.java

@@ -1,15 +1,12 @@
 package org.embulk.spi.type;
 
-import java.util.HashMap;
 import java.util.Map;
 import java.io.IOException;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableMap;
-import com.fasterxml.jackson.databind.module.SimpleModule;
 import com.fasterxml.jackson.databind.DeserializationContext;
 import com.fasterxml.jackson.databind.deser.std.FromStringDeserializer;
 import com.fasterxml.jackson.databind.JsonMappingException;
-import org.embulk.config.ModelManager;
 
 public class TypeDeserializer
         extends FromStringDeserializer<Type>

data/embulk-core/src/main/java/org/embulk/spi/unit/LocalFileSerDe.java

@@ -11,9 +11,6 @@ import com.fasterxml.jackson.databind.JsonSerializer;
 import com.fasterxml.jackson.databind.JsonDeserializer;
 import com.fasterxml.jackson.databind.SerializerProvider;
 import com.fasterxml.jackson.databind.DeserializationContext;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.JsonMappingException;
-import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.module.guice.ObjectMapperModule;
 
 public class LocalFileSerDe

data/embulk-core/src/main/java/org/embulk/spi/unit/ToStringMap.java

@@ -6,7 +6,6 @@ import java.util.Properties;
 import com.google.common.base.Function;
 import com.google.common.collect.Maps;
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonValue;
 
 public class ToStringMap
         extends HashMap<String, String>

data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java

@@ -2,7 +2,7 @@ package org.embulk.spi.util;
 
 import java.util.List;
 import org.embulk.config.TaskSource;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.ExecSession;
 import org.embulk.spi.ProcessState;
 import org.embulk.spi.Schema;
@@ -12,6 +12,7 @@ import org.embulk.spi.InputPlugin;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.ProcessTask;
+import org.embulk.plugin.compat.PluginWrappers;
 
 public abstract class Executors
 {
@@ -21,9 +22,9 @@ public abstract class Executors
     {
         public void started();
 
-        public void inputCommitted(CommitReport report);
+        public void inputCommitted(TaskReport report);
 
-        public void outputCommitted(CommitReport report);
+        public void outputCommitted(TaskReport report);
     }
 
     public static void process(ExecSession exec,
@@ -49,25 +50,27 @@ public abstract class Executors
             OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource,
             ProcessStateCallback callback)
     {
-        TransactionalPageOutput tran = outputPlugin.open(outputTaskSource, outputSchema, taskIndex);
+        TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
+            outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
 
         PageOutput closeThis = tran;
         callback.started();
         try {
             PageOutput filtered = closeThis = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
 
-            CommitReport inputCommitReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
-            if (inputCommitReport == null) {
-                inputCommitReport = exec.newCommitReport();
+            TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
+
+            if (inputTaskReport == null) {
+                inputTaskReport = exec.newTaskReport();
             }
-            callback.inputCommitted(inputCommitReport);
+            callback.inputCommitted(inputTaskReport);
 
-            CommitReport outputCommitReport = tran.commit();
+            TaskReport outputTaskReport = tran.commit();
             tran = null;
-            if (outputCommitReport == null) {
-                outputCommitReport = exec.newCommitReport();
+            if (outputTaskReport == null) {
+                outputTaskReport = exec.newTaskReport();
             }
-            callback.outputCommitted(outputCommitReport);  // TODO check output.finish() is called. wrap or abstract
+            callback.outputCommitted(outputTaskReport);  // TODO check output.finish() is called. wrap or abstract
 
         } finally {
             try {