embulk 0.6.27 → 0.7.0

data/embulk-core/src/main/java/org/embulk/exec/LocalExecutorPlugin.java

@@ -9,12 +9,11 @@ import java.util.concurrent.ExecutionException;
 import org.slf4j.Logger;
 import com.google.inject.Inject;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecutorPlugin;
 import org.embulk.spi.ProcessTask;
 import org.embulk.spi.ProcessState;
-import org.embulk.spi.TaskState;
 import org.embulk.spi.Schema;
 import org.embulk.spi.util.Executors;
 import org.embulk.spi.util.Executors.ProcessStateCallback;
@@ -109,14 +108,14 @@ public class LocalExecutorPlugin
                             state.getOutputTaskState(taskIndex).start();
                         }
 
-                        public void inputCommitted(CommitReport report)
+                        public void inputCommitted(TaskReport report)
                         {
-                            state.getInputTaskState(taskIndex).setCommitReport(report);
+                            state.getInputTaskState(taskIndex).setTaskReport(report);
                         }
 
-                        public void outputCommitted(CommitReport report)
+                        public void outputCommitted(TaskReport report)
                         {
-                            state.getOutputTaskState(taskIndex).setCommitReport(report);
+                            state.getOutputTaskState(taskIndex).setTaskReport(report);
                         }
                     });
                     return null;

data/embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java

@@ -1,6 +1,5 @@
 package org.embulk.exec;
 
-import java.util.Properties;
 import org.slf4j.ILoggerFactory;
 import org.slf4j.LoggerFactory;
 import ch.qos.logback.classic.Level;

data/embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java

@@ -2,9 +2,6 @@ package org.embulk.exec;
 
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.buffer.ByteBuf;
-import io.netty.util.ResourceLeakDetector;
-import io.netty.util.internal.logging.InternalLoggerFactory;
-import io.netty.util.internal.logging.JdkLoggerFactory;
 import org.embulk.spi.Buffer;
 import org.embulk.spi.BufferAllocator;
 import com.google.inject.Inject;

data/embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java

@@ -11,7 +11,7 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.plugin.PluginType;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Page;
@@ -91,7 +91,7 @@ public class PreviewExecutor
 
         try {
             input.transaction(task.getInputConfig(), new InputPlugin.Control() {
-                public List<CommitReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount)
+                public List<TaskReport> run(final TaskSource inputTask, Schema inputSchema, final int taskCount)
                 {
                     Filters.transaction(filterPlugins, task.getFilterConfigs(), inputSchema, new Filters.Control() {
                         public void run(final List<TaskSource> filterTasks, final List<Schema> filterSchemas)

data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java

@@ -4,11 +4,12 @@ import java.util.List;
 import com.google.common.base.Optional;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
 import org.embulk.config.CommitReport;
 import org.embulk.spi.Schema;
-import org.embulk.spi.ExecSession;
 
 public class ResumeState
 {
@@ -17,8 +18,8 @@ public class ResumeState
     private final TaskSource outputTaskSource;
     private final Schema inputSchema;
     private final Schema outputSchema;
-    private final List<Optional<CommitReport>> inputCommitReports;
-    private final List<Optional<CommitReport>> outputCommitReports;
+    private final List<Optional<TaskReport>> inputTaskReports;
+    private final List<Optional<TaskReport>> outputTaskReports;
 
     @JsonCreator
     public ResumeState(
@@ -27,16 +28,16 @@ public class ResumeState
             @JsonProperty("out_task") TaskSource outputTaskSource,
             @JsonProperty("in_schema") Schema inputSchema,
             @JsonProperty("out_schema") Schema outputSchema,
-            @JsonProperty("in_reports") List<Optional<CommitReport>> inputCommitReports,
-            @JsonProperty("out_reports") List<Optional<CommitReport>> outputCommitReports)
+            @JsonProperty("in_reports") List<Optional<TaskReport>> inputTaskReports,
+            @JsonProperty("out_reports") List<Optional<TaskReport>> outputTaskReports)
     {
         this.execSessionConfigSource = execSessionConfigSource;
         this.inputTaskSource = inputTaskSource;
         this.outputTaskSource = outputTaskSource;
         this.inputSchema = inputSchema;
         this.outputSchema = outputSchema;
-        this.inputCommitReports = inputCommitReports;
-        this.outputCommitReports = outputCommitReports;
+        this.inputTaskReports = inputTaskReports;
+        this.outputTaskReports = outputTaskReports;
     }
 
     @JsonProperty("exec_task")
@@ -70,14 +71,30 @@ public class ResumeState
     }
 
     @JsonProperty("in_reports")
+    public List<Optional<TaskReport>> getInputTaskReports()
+    {
+        return inputTaskReports;
+    }
+
+    @Deprecated
+    @JsonIgnore
+    @SuppressWarnings("unchecked")
     public List<Optional<CommitReport>> getInputCommitReports()
     {
-        return inputCommitReports;
+        return (List) inputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport
     }
 
     @JsonProperty("out_reports")
+    public List<Optional<TaskReport>> getOutputTaskReports()
+    {
+        return outputTaskReports;
+    }
+
+    @Deprecated
+    @JsonIgnore
+    @SuppressWarnings("unchecked")
     public List<Optional<CommitReport>> getOutputCommitReports()
     {
-        return outputCommitReports;
+        return (List) outputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport;
     }
 }

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -5,8 +5,7 @@ import com.google.inject.Inject;
 import com.google.common.base.Preconditions;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
-import org.embulk.config.CommitReport;
-import org.embulk.plugin.PluginType;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Exec;
 import org.embulk.spi.Page;
@@ -16,7 +15,6 @@ import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.FileInputRunner;
 import org.embulk.spi.PageOutput;
-import org.embulk.exec.ForSystemConfig;
 import static org.embulk.spi.util.Inputs.each;
 
 /*
@@ -34,7 +32,7 @@ public class SamplingParserPlugin
 
         try {
             runner.transaction(samplingInputConfig, new InputPlugin.Control() {
-                public List<CommitReport> run(TaskSource taskSource, Schema schema, int taskCount)
+                public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
                 {
                     if (taskCount == 0) {
                         throw new NoSampleException("No input files to read sample data");

data/embulk-core/src/main/java/org/embulk/guice/Bootstrap.java

@@ -100,17 +100,24 @@ public class Bootstrap
         return this;
     }
 
-    public Injector initialize()
+    public LifeCycleInjector initialize()
     {
-        Injector injector = start();
-        injector.getInstance(LifeCycleManager.class).destroyOnShutdownHook();
-        return injector;
+        return build(true);
     }
 
     public CloseableInjector initializeCloseable()
+    {
+        return build(false);
+    }
+
+    private LifeCycleInjectorProxy build(boolean destroyOnShutdownHook)
     {
         Injector injector = start();
-        return new CloseableInjectorProxy(injector, injector.getInstance(LifeCycleManager.class));
+        LifeCycleManager lifeCycleManager = injector.getInstance(LifeCycleManager.class);
+        if (destroyOnShutdownHook) {
+            lifeCycleManager.destroyOnShutdownHook();
+        }
+        return new LifeCycleInjectorProxy(injector, lifeCycleManager);
     }
 
     private Injector start()

data/embulk-core/src/main/java/org/embulk/guice/CloseableInjector.java

@@ -18,5 +18,5 @@ package org.embulk.guice;
 import com.google.inject.Injector;
 
 public interface CloseableInjector
-        extends Injector, AutoCloseable
+        extends LifeCycleInjector, AutoCloseable
 { }

data/embulk-core/src/main/java/org/embulk/guice/LifeCycleInjector.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright 2015 Sadayuki Furuhashi
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.embulk.guice;
+
+import com.google.inject.Injector;
+
+public interface LifeCycleInjector
+        extends Injector
+{
+    boolean isDestroyed();
+
+    void destroy() throws Exception;
+}

data/embulk-core/src/main/java/org/embulk/guice/{CloseableInjectorProxy.java → LifeCycleInjectorProxy.java}

@@ -17,14 +17,14 @@ package org.embulk.guice;
 
 import com.google.inject.Injector;
 
-class CloseableInjectorProxy
+class LifeCycleInjectorProxy
         extends InjectorProxy
-        implements CloseableInjector
+        implements LifeCycleInjector, CloseableInjector
 {
     private final Injector injector;
     private final LifeCycleManager lifeCycleManager;
 
-    public CloseableInjectorProxy(Injector injector, LifeCycleManager lifeCycleManager)
+    public LifeCycleInjectorProxy(Injector injector, LifeCycleManager lifeCycleManager)
     {
         this.injector = injector;
         this.lifeCycleManager = lifeCycleManager;
@@ -33,15 +33,29 @@ class CloseableInjectorProxy
     @Override
     protected synchronized Injector injector()
     {
-        if (lifeCycleManager.isDestroyed()) {
-            throw new IllegalStateException("Injector already closed");
+        if (isDestroyed()) {
+            throw new IllegalStateException("Injector already destroyed");
         }
         return injector;
     }
 
     @Override
-    public synchronized void close() throws Exception
+    public synchronized boolean isDestroyed()
+    {
+        return lifeCycleManager.isDestroyed();
+    }
+
+    @Override
+    public synchronized void destroy()
+            throws Exception
     {
         lifeCycleManager.destroy();  // reentrant
     }
+
+    @Override
+    public void close()
+            throws Exception
+    {
+        destroy();
+    }
 }

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -3,13 +3,11 @@ package org.embulk.jruby;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Set;
-import java.io.File;
 import org.slf4j.ILoggerFactory;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;
-import com.google.inject.Provider;
 import com.google.inject.multibindings.Multibinder;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
@@ -67,16 +65,19 @@ public class JRubyScriptingModule
             jruby.setCompatVersion(CompatVersion.RUBY1_9);
 
             // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
-            // $LOAD_PATH is set by lib/embulk/command/embulk.rb if Embulk starts
+            // $LOAD_PATH is set by lib/embulk/command/embulk_run.rb if Embulk starts
             // using embulk-cli but it's not set if Embulk is embedded in an application.
             // Here adds this jar's internal resources to $LOAD_PATH for those applciations.
 
-            List<String> loadPaths = new ArrayList<String>(jruby.getLoadPaths());
-            String coreJarPath = JRubyScriptingModule.class.getProtectionDomain().getCodeSource().getLocation().getPath();
-            if (!loadPaths.contains(coreJarPath)) {
-                loadPaths.add(coreJarPath);
-            }
-            jruby.setLoadPaths(loadPaths);
+//            List<String> loadPaths = new ArrayList<String>(jruby.getLoadPaths());
+//            String coreJarPath = JRubyScriptingModule.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+//            if (!loadPaths.contains(coreJarPath)) {
+//                loadPaths.add(coreJarPath);
+//            }
+//            jruby.setLoadPaths(loadPaths);
+
+            // load embulk.rb
+            jruby.runScriptlet("require 'embulk'");
 
             // jruby searches embulk/java/bootstrap.rb from the beginning of $LOAD_PATH.
             jruby.runScriptlet("require 'embulk/java/bootstrap'");
@@ -96,9 +97,6 @@ public class JRubyScriptingModule
                     jruby.runScriptlet("Embulk::Java::Injected"),
                     "const_set", "BufferAllocator", injector.getInstance(BufferAllocator.class));
 
-            // load embulk.rb
-            jruby.runScriptlet("require 'embulk'");
-
             // initialize logger
             jruby.callMethod(
                     jruby.runScriptlet("Embulk"),

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoaderModule.java

@@ -5,7 +5,6 @@ import java.util.Properties;
 import java.net.URL;
 import java.io.InputStream;
 import java.io.IOException;
-import com.google.inject.Inject;
 import com.google.inject.Module;
 import com.google.inject.Binder;
 import com.google.inject.Scopes;

data/embulk-core/src/main/java/org/embulk/plugin/PluginManager.java

@@ -7,6 +7,8 @@ import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
 import com.google.inject.Injector;
 import org.embulk.config.ConfigException;
+import org.embulk.spi.InputPlugin;
+import org.embulk.plugin.compat.PluginWrappers;
 
 public class PluginManager
 {
@@ -22,7 +24,17 @@ public class PluginManager
         this.injector = injector;
     }
 
+    @SuppressWarnings("unchecked")
     public <T> T newPlugin(Class<T> iface, PluginType type)
+    {
+        T t = newPluginWithoutWrapper(iface, type);
+        if (t instanceof InputPlugin) {
+            return (T) PluginWrappers.inputPlugin((InputPlugin) t);
+        }
+        return t;
+    }
+
+    private <T> T newPluginWithoutWrapper(Class<T> iface, PluginType type)
     {
         if (sources.isEmpty()) {
             throw new ConfigException("No PluginSource is installed");
@@ -36,7 +48,8 @@ public class PluginManager
         for (PluginSource source : sources) {
             try {
                 return source.newPlugin(iface, type);
-            } catch (PluginSourceNotMatchException e) {
+            }
+            catch (PluginSourceNotMatchException e) {
                 exceptions.add(e);
             }
         }

data/embulk-core/src/main/java/org/embulk/plugin/compat/InputPluginWrapper.java

@@ -0,0 +1,102 @@
+package org.embulk.plugin.compat;
+
+import java.util.List;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+import com.google.common.base.Throwables;
+import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.InputPlugin;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.TaskSource;
+
+public class InputPluginWrapper
+        implements InputPlugin
+{
+    public static InputPlugin wrapIfNecessary(InputPlugin object)
+    {
+        Method runMethod = wrapRunMethod(object);
+        if (runMethod != null) {
+            return new InputPluginWrapper(object, runMethod);
+        }
+        return object;
+    }
+
+    private static Method wrapRunMethod(InputPlugin object)
+    {
+        try {
+            Method m = object.getClass().getMethod("run", TaskSource.class, Schema.class, int.class, PageOutput.class);
+            if (m.getReturnType().equals(CommitReport.class)) {
+                return m;
+            } else {
+                return null;
+            }
+        }
+        catch (NoSuchMethodException ex) {
+            return null;
+        }
+    }
+
+    private final InputPlugin object;
+    private final Method runMethod;
+
+    private InputPluginWrapper(InputPlugin object,
+            Method runMethod)
+    {
+        this.object = object;
+        this.runMethod = runMethod;
+    }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+            InputPlugin.Control control)
+    {
+        return object.transaction(config, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            InputPlugin.Control control)
+    {
+        return object.resume(taskSource, schema, taskCount, control);
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int taskCount,
+            List<TaskReport> successTaskReports)
+    {
+        object.cleanup(taskSource, schema, taskCount, successTaskReports);
+    }
+
+    @Override
+    public TaskReport run(TaskSource taskSource,
+            Schema schema, int taskIndex,
+            PageOutput output)
+    {
+        if (runMethod != null) {
+            try {
+                return (TaskReport) runMethod.invoke(object, taskSource, schema, taskIndex, output);
+            }
+            catch (IllegalAccessException | IllegalArgumentException ex) {
+                throw Throwables.propagate(ex);
+            }
+            catch (InvocationTargetException ex) {
+                throw Throwables.propagate(ex.getCause());
+            }
+
+        } else {
+            return object.run(taskSource, schema, taskIndex, output);
+        }
+    }
+
+    @Override
+    public ConfigDiff guess(ConfigSource config)
+    {
+        return object.guess(config);
+    }
+}