embulk 0.6.27 → 0.7.0

data/embulk-core/src/main/java/org/embulk/plugin/compat/PluginWrappers.java

@@ -0,0 +1,30 @@
+package org.embulk.plugin.compat;
+
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.plugin.compat.InputPluginWrapper;
+
+public class PluginWrappers
+{
+    public static InputPlugin inputPlugin(InputPlugin input)
+    {
+        return InputPluginWrapper.wrapIfNecessary(input);
+    }
+
+    public static TransactionalFileInput transactionalFileInput(TransactionalFileInput tran)
+    {
+        return TransactionalFileInputWrapper.wrapIfNecessary(tran);
+    }
+
+    public static TransactionalFileOutput transactionalFileOutput(TransactionalFileOutput tran)
+    {
+        return TransactionalFileOutputWrapper.wrapIfNecessary(tran);
+    }
+
+    public static TransactionalPageOutput transactionalPageOutput(TransactionalPageOutput tran)
+    {
+        return TransactionalPageOutputWrapper.wrapIfNecessary(tran);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileInputWrapper.java

@@ -0,0 +1,96 @@
+package org.embulk.plugin.compat;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.TaskReport;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.TransactionalFileInput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TransactionalFileInputWrapper
+        implements TransactionalFileInput
+{
+    private final Logger logger = LoggerFactory.getLogger(PluginWrappers.class);
+
+    public static TransactionalFileInput wrapIfNecessary(TransactionalFileInput object)
+    {
+        Method runMethod = wrapCommitMethod(object);
+        if (runMethod != null) {
+            return new TransactionalFileInputWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapCommitMethod(TransactionalFileInput object)
+    {
+        try {
+            Method m = object.getClass().getMethod("commit");
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final TransactionalFileInput object;
+    private final Method commitMethod;
+
+    private TransactionalFileInputWrapper(TransactionalFileInput object,
+            Method commitMethod)
+    {
+        this.object = object;
+        this.commitMethod = commitMethod;
+        logger.warn("A file input plugin is compiled with old Embulk plugin API. Please update the plugin version using \"embulk gem install\" command, or contact a developer of the plugin to upgrade the plugin code using \"embulk migrate\" command: {}", object.getClass());
+    }
+
+    @Override
+    public Buffer poll()
+    {
+        return object.poll();
+    }
+
+    @Override
+    public boolean nextFile()
+    {
+        return object.nextFile();
+    }
+
+    @Override
+    public void close()
+    {
+        object.close();
+    }
+
+    @Override
+    public void abort()
+    {
+        object.abort();
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        if (commitMethod != null) {
+            try {
+                return (TaskReport) commitMethod.invoke(object);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.commit();
+        }
+    }
+}
+

data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalFileOutputWrapper.java

@@ -0,0 +1,102 @@
+package org.embulk.plugin.compat;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.TaskReport;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.TransactionalFileOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TransactionalFileOutputWrapper
+        implements TransactionalFileOutput
+{
+    private final Logger logger = LoggerFactory.getLogger(PluginWrappers.class);
+
+    public static TransactionalFileOutput wrapIfNecessary(TransactionalFileOutput object)
+    {
+        Method runMethod = wrapCommitMethod(object);
+        if (runMethod != null) {
+            return new TransactionalFileOutputWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapCommitMethod(TransactionalFileOutput object)
+    {
+        try {
+            Method m = object.getClass().getMethod("commit");
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final TransactionalFileOutput object;
+    private final Method commitMethod;
+
+    private TransactionalFileOutputWrapper(TransactionalFileOutput object,
+            Method commitMethod)
+    {
+        this.object = object;
+        this.commitMethod = commitMethod;
+        logger.warn("A file output plugin is compiled with old Embulk plugin API. Please update the plugin version using \"embulk gem install\" command, or contact a developer of the plugin to upgrade the plugin code using \"embulk migrate\" command: {}", object.getClass());
+    }
+
+    @Override
+    public void nextFile()
+    {
+        object.nextFile();
+    }
+
+    @Override
+    public void add(Buffer buffer)
+    {
+        object.add(buffer);
+    }
+
+    @Override
+    public void finish()
+    {
+        object.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        object.close();
+    }
+
+    @Override
+    public void abort()
+    {
+        object.abort();
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        if (commitMethod != null) {
+            try {
+                return (TaskReport) commitMethod.invoke(object);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.commit();
+        }
+    }
+}
+

data/embulk-core/src/main/java/org/embulk/plugin/compat/TransactionalPageOutputWrapper.java

@@ -0,0 +1,95 @@
+package org.embulk.plugin.compat;
+
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.TaskReport;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.Page;
+import org.embulk.spi.TransactionalPageOutput;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TransactionalPageOutputWrapper
+        implements TransactionalPageOutput
+{
+    private final Logger logger = LoggerFactory.getLogger(PluginWrappers.class);
+
+    public static TransactionalPageOutput wrapIfNecessary(TransactionalPageOutput object)
+    {
+        Method runMethod = wrapCommitMethod(object);
+        if (runMethod != null) {
+            return new TransactionalPageOutputWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapCommitMethod(TransactionalPageOutput object)
+    {
+        try {
+            Method m = object.getClass().getMethod("commit");
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final TransactionalPageOutput object;
+    private final Method commitMethod;
+
+    private TransactionalPageOutputWrapper(TransactionalPageOutput object,
+            Method commitMethod)
+    {
+        this.object = object;
+        this.commitMethod = commitMethod;
+        logger.warn("An output plugin is compiled with old Embulk plugin API. Please update the plugin version using \"embulk gem install\" command, or contact a developer of the plugin to upgrade the plugin code using \"embulk migrate\" command: {}", object.getClass());
+    }
+
+    @Override
+    public void add(Page page)
+    {
+        object.add(page);
+    }
+
+    @Override
+    public void finish()
+    {
+        object.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        object.close();
+    }
+
+    @Override
+    public void abort()
+    {
+        object.abort();
+    }
+
+    @Override
+    public TaskReport commit()
+    {
+        if (commitMethod != null) {
+            try {
+                return (TaskReport) commitMethod.invoke(object);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.commit();
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/ColumnConfig.java

@@ -2,12 +2,10 @@ package org.embulk.spi;
 
 import java.util.Objects;
 import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
 import com.fasterxml.jackson.annotation.JsonValue;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.type.Type;
 import org.embulk.spi.type.TimestampType;
-import org.embulk.spi.Exec;
 
 public class ColumnConfig
 {

data/embulk-core/src/main/java/org/embulk/spi/Exec.java

@@ -3,9 +3,9 @@ package org.embulk.spi;
 import java.util.concurrent.ExecutionException;
 import org.slf4j.Logger;
 import com.google.inject.Injector;
-import org.embulk.config.Task;
 import org.embulk.config.ModelManager;
-import org.embulk.config.CommitReport;
+import org.embulk.config.CommitReport;  // Deprecated
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
@@ -74,6 +74,12 @@ public class Exec
         return session().newPlugin(iface, type);
     }
 
+    public static TaskReport newTaskReport()
+    {
+        return session().newTaskReport();
+    }
+
+    @Deprecated
     public static CommitReport newCommitReport()
     {
         return session().newCommitReport();

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -1,6 +1,5 @@
 package org.embulk.spi;
 
-import java.io.File;
 import org.joda.time.DateTimeZone;
 import org.slf4j.Logger;
 import org.slf4j.ILoggerFactory;
@@ -11,6 +10,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ModelManager;
 import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
@@ -166,6 +166,12 @@ public class ExecSession
         return pluginManager.newPlugin(iface, type);
     }
 
+    public TaskReport newTaskReport()
+    {
+        return new DataSourceImpl(modelManager);
+    }
+
+    @Deprecated
     public CommitReport newCommitReport()
     {
         return new DataSourceImpl(modelManager);

data/embulk-core/src/main/java/org/embulk/spi/Extension.java

@@ -8,10 +8,10 @@ import org.embulk.config.ConfigSource;
  * Extension is a module to extend the execution framework using Guice.
  * Unlike plugins, extensions can overwrite or add core components such as
  * BufferManager, PluginSource, etc.
- * Extension is not designed for users but for framework developpers to make
+ * Extension is not designed for users but for framework developers to make
  * core components loosely coupled.
  *
- * An example extention to add a custom PluginSource will be as following:
+ * An example extension to add a custom PluginSource will be as following:
  *
  * <code>
  * class MyPluginSourceExtension

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -4,13 +4,13 @@ import java.util.List;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 
 public interface FileInputPlugin
 {
     interface Control
     {
-        List<CommitReport> run(TaskSource taskSource,
+        List<TaskReport> run(TaskSource taskSource,
                 int taskCount);
     }
 
@@ -23,7 +23,7 @@ public interface FileInputPlugin
 
     void cleanup(TaskSource taskSource,
             int taskCount,
-            List<CommitReport> successCommitReports);
+            List<TaskReport> successTaskReports);
 
     TransactionalFileInput open(TaskSource taskSource,
             int taskIndex);

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -6,10 +6,11 @@ import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.plugin.PluginType;
+import org.embulk.plugin.compat.PluginWrappers;
 import org.embulk.spi.util.Decoders;
 import org.embulk.exec.GuessExecutor;
 import org.embulk.exec.SamplingParserPlugin;
@@ -102,9 +103,9 @@ public class FileInputRunner
         }
 
         @Override
-        public List<CommitReport> run(final TaskSource fileInputTaskSource, final int taskCount)
+        public List<TaskReport> run(final TaskSource fileInputTaskSource, final int taskCount)
         {
-            final List<CommitReport> commitReports = new ArrayList<CommitReport>();
+            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
             Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
                 public void run(final List<TaskSource> decoderTaskSources)
                 {
@@ -114,37 +115,38 @@ public class FileInputRunner
                             task.setFileInputTaskSource(fileInputTaskSource);
                             task.setDecoderTaskSources(decoderTaskSources);
                             task.setParserTaskSource(parserTaskSource);
-                            commitReports.addAll(nextControl.run(task.dump(), schema, taskCount));
+                            taskReports.addAll(nextControl.run(task.dump(), schema, taskCount));
                         }
                     });
                 }
             });
-            return commitReports;
+            return taskReports;
         }
     }
 
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successTaskReports)
     {
-        fileInputPlugin.cleanup(taskSource, taskCount, successCommitReports);
+        fileInputPlugin.cleanup(taskSource, taskCount, successTaskReports);
     }
 
     @Override
-    public CommitReport run(TaskSource taskSource, Schema schema, int taskIndex,
+    public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex,
             PageOutput output)
     {
         final RunnerTask task = taskSource.loadTask(RunnerTask.class);
         List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
         ParserPlugin parserPlugin = newParserPlugin(task);
 
-        TransactionalFileInput tran = fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex);
+        TransactionalFileInput tran = PluginWrappers.transactionalFileInput(
+                fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex));
         FileInput fileInput = tran;
         try {
             fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), fileInput);
             parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
 
-            CommitReport report = tran.commit();  // TODO check output.finish() is called. wrap
+            TaskReport report = tran.commit();  // TODO check output.finish() is called. wrap
             tran = null;
             return report;
         } finally {