embulk 0.5.5 → 0.6.0

data/embulk-core/src/main/java/org/embulk/spi/ProcessState.java

@@ -0,0 +1,10 @@
+package org.embulk.spi;
+
+public interface ProcessState
+{
+    public void initialize(int inputTaskCount, int outputTaskCount);
+
+    public TaskState getInputTaskState(int inputTaskIndex);
+
+    public TaskState getOutputTaskState(int outputTaskIndex);
+}

data/embulk-core/src/main/java/org/embulk/spi/ProcessTask.java

@@ -0,0 +1,118 @@
+package org.embulk.spi;
+
+import java.util.List;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.embulk.plugin.PluginType;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Schema;
+import org.embulk.spi.util.Executors;
+
+public class ProcessTask
+{
+    private final PluginType inputPluginType;
+    private final PluginType outputPluginType;
+    private final List<PluginType> filterPluginTypes;
+    private final TaskSource inputTaskSource;
+    private final TaskSource outputTaskSource;
+    private final List<TaskSource> filterTaskSources;
+    private final List<Schema> schemas;
+    private final Schema executorSchema;
+    private TaskSource executorTaskSource;
+
+    @JsonCreator
+    public ProcessTask(
+            @JsonProperty("inputType") PluginType inputPluginType,
+            @JsonProperty("outputType") PluginType outputPluginType,
+            @JsonProperty("filterTypes") List<PluginType> filterPluginTypes,
+            @JsonProperty("inputTask") TaskSource inputTaskSource,
+            @JsonProperty("outputTask") TaskSource outputTaskSource,
+            @JsonProperty("filterTasks") List<TaskSource> filterTaskSources,
+            @JsonProperty("schemas") List<Schema> schemas,
+            @JsonProperty("executorSchema") Schema executorSchema,
+            @JsonProperty("executorTask") TaskSource executorTaskSource)
+    {
+        this.inputPluginType = inputPluginType;
+        this.outputPluginType = outputPluginType;
+        this.filterPluginTypes = filterPluginTypes;
+        this.inputTaskSource = inputTaskSource;
+        this.outputTaskSource = outputTaskSource;
+        this.filterTaskSources = filterTaskSources;
+        this.schemas = schemas;
+        this.executorSchema = executorSchema;
+        this.executorTaskSource = executorTaskSource;
+    }
+
+    @JsonProperty("inputType")
+    public PluginType getInputPluginType()
+    {
+        return inputPluginType;
+    }
+
+    @JsonProperty("outputType")
+    public PluginType getOutputPluginType()
+    {
+        return outputPluginType;
+    }
+
+    @JsonProperty("filterTypes")
+    public List<PluginType> getFilterPluginTypes()
+    {
+        return filterPluginTypes;
+    }
+
+    @JsonProperty("inputTask")
+    public TaskSource getInputTaskSource()
+    {
+        return inputTaskSource;
+    }
+
+    @JsonProperty("outputTask")
+    public TaskSource getOutputTaskSource()
+    {
+        return outputTaskSource;
+    }
+
+    @JsonProperty("filterTasks")
+    public List<TaskSource> getFilterTaskSources()
+    {
+        return filterTaskSources;
+    }
+
+    @JsonProperty("schemas")
+    public List<Schema> getFilterSchemas()
+    {
+        return schemas;
+    }
+
+    @JsonProperty("executorSchema")
+    public Schema getExecutorSchema()
+    {
+        return executorSchema;
+    }
+
+    @JsonIgnore
+    public Schema getInputSchema()
+    {
+        return Executors.getInputSchema(schemas);
+    }
+
+    @JsonIgnore
+    public Schema getOutputSchema()
+    {
+        return Executors.getOutputSchema(schemas);
+    }
+
+    @JsonIgnore
+    public void setExecutorTaskSource(TaskSource executorTaskSource)
+    {
+        this.executorTaskSource = executorTaskSource;
+    }
+
+    @JsonProperty("executorTask")
+    public TaskSource getExecutorTaskSource()
+    {
+        return executorTaskSource;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/TaskState.java

@@ -0,0 +1,70 @@
+package org.embulk.spi;
+
+import com.google.common.base.Optional;
+import org.embulk.config.CommitReport;
+
+public class TaskState
+{
+    private volatile boolean started = false;
+    private volatile boolean finished = false;
+    private volatile Optional<CommitReport> commitReport = Optional.absent();
+    private volatile Optional<Throwable> exception = Optional.absent();
+
+    public void start()
+    {
+        this.started = true;
+    }
+
+    public void finish()
+    {
+        this.started = true;
+        this.finished = true;
+    }
+
+    public void setCommitReport(CommitReport commitReport)
+    {
+        this.started = true;
+        this.commitReport = Optional.of(commitReport);
+    }
+
+    public void setException(Throwable exception)
+    {
+        this.started = true;
+        if (exception == null) {
+            this.exception = Optional.absent();
+        } else {
+            this.exception = Optional.of(exception);
+        }
+    }
+
+    public void resetException()
+    {
+        this.started = true;
+        this.exception = Optional.absent();
+    }
+
+    public boolean isStarted()
+    {
+        return started;
+    }
+
+    public boolean isFinished()
+    {
+        return finished;
+    }
+
+    public boolean isCommitted()
+    {
+        return commitReport.isPresent();
+    }
+
+    public Optional<CommitReport> getCommitReport()
+    {
+        return commitReport;
+    }
+
+    public Optional<Throwable> getException()
+    {
+        return exception;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java

@@ -0,0 +1,92 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.ProcessState;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ProcessTask;
+
+public abstract class Executors
+{
+    private Executors() { }
+
+    public interface ProcessStateCallback
+    {
+        public void started();
+
+        public void inputCommitted(CommitReport report);
+
+        public void outputCommitted(CommitReport report);
+    }
+
+    public static void process(ExecSession exec,
+            ProcessTask task, int taskIndex,
+            ProcessStateCallback callback)
+    {
+        InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
+        List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
+        OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
+
+        // TODO assert task.getExecutorSchema().equals task.getOutputSchema()
+
+        process(exec, taskIndex,
+                inputPlugin, task.getInputSchema(), task.getInputTaskSource(),
+                filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources(),
+                outputPlugin, task.getOutputSchema(), task.getOutputTaskSource(),
+                callback);
+    }
+
+    public static void process(ExecSession exec, int taskIndex,
+            InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource,
+            List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources,
+            OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource,
+            ProcessStateCallback callback)
+    {
+        TransactionalPageOutput tran = outputPlugin.open(outputTaskSource, outputSchema, taskIndex);
+
+        PageOutput closeThis = tran;
+        callback.started();
+        try {
+            PageOutput filtered = closeThis = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
+
+            CommitReport inputCommitReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
+            if (inputCommitReport == null) {
+                inputCommitReport = exec.newCommitReport();
+            }
+            callback.inputCommitted(inputCommitReport);
+
+            CommitReport outputCommitReport = tran.commit();
+            tran = null;
+            if (outputCommitReport == null) {
+                outputCommitReport = exec.newCommitReport();
+            }
+            callback.outputCommitted(outputCommitReport);  // TODO check output.finish() is called. wrap or abstract
+
+        } finally {
+            try {
+                if (tran != null) {
+                    tran.abort();
+                }
+            } finally {
+                closeThis.close();
+            }
+        }
+    }
+
+    public static Schema getInputSchema(List<Schema> schemas)
+    {
+        return schemas.get(0);
+    }
+
+    public static Schema getOutputSchema(List<Schema> schemas)
+    {
+        return schemas.get(schemas.size() - 1);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -16,11 +16,25 @@ public abstract class Filters
 {
     private Filters() { }
 
-    public static List<FilterPlugin> newFilterPlugins(ExecSession exec, List<ConfigSource> configs)
+    public static List<PluginType> getPluginTypes(List<ConfigSource> configs)
     {
-        ImmutableList.Builder<FilterPlugin> builder = ImmutableList.builder();
+        ImmutableList.Builder<PluginType> builder = ImmutableList.builder();
         for (ConfigSource config : configs) {
-            builder.add(exec.newPlugin(FilterPlugin.class, config.get(PluginType.class, "type")));
+            builder.add(config.get(PluginType.class, "type"));
+        }
+        return builder.build();
+    }
+
+    public static List<FilterPlugin> newFilterPluginsFromConfigSources(ExecSession exec, List<ConfigSource> configs)
+    {
+        return newFilterPlugins(exec, getPluginTypes(configs));
+    }
+
+    public static List<FilterPlugin> newFilterPlugins(ExecSession exec, List<PluginType> pluginTypes)
+    {
+        ImmutableList.Builder<FilterPlugin> builder = ImmutableList.builder();
+        for (PluginType pluginType : pluginTypes) {
+            builder.add(exec.newPlugin(FilterPlugin.class, pluginType));
         }
         return builder.build();
     }

data/embulk-core/src/test/java/org/embulk/spi/TestBuffer.java

@@ -0,0 +1,24 @@
+package org.embulk.spi;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import org.junit.Test;
+
+public class TestBuffer
+{
+    @Test
+    public void testEquals() throws Exception
+    {
+        byte[] bytes = new byte[] { 1, 2, 3, 2, 3 };
+        Buffer b1 = Buffer.wrap(bytes, 0, 2);  // [1, 2]
+        Buffer b2 = Buffer.wrap(bytes, 1, 2);  // [2, 3]
+        Buffer b3 = Buffer.wrap(bytes, 3, 2);  // [2, 3]
+
+        assertFalse(b1.equals(b2));
+        assertTrue(b2.equals(b3));
+
+        assertFalse(b1.hashCode() == b2.hashCode());
+        assertTrue(b2.hashCode() == b3.hashCode());
+    }
+}

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -56,7 +56,7 @@ You can find the latest embulk binary from the `releases <https://bintray.com/em
 
 .. code-block:: console
 
-    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.5.5.jar -O /usr/local/bin/embulk
+    $ sudo wget https://bintray.com/artifact/download/embulk/maven/embulk-0.6.0.jar -O /usr/local/bin/embulk
     $ sudo chmod +x /usr/local/bin/embulk
 
 Step 2. Install Elasticsearch plugin

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.0
     release/release-0.5.5
     release/release-0.5.4
     release/release-0.5.3

data/embulk-docs/src/release/release-0.6.0.rst

@@ -0,0 +1,34 @@
+Release 0.6.0
+==================================
+
+Executor Plugin Mechanism
+------------------
+
+Now executor of Embulk is fully extensible using plugins. Executor plugins get input, filter and output plugins from the Embulk framework and runs them using multiple threads, processes, or servers. While input, filter and output plugins are response for data processing, executor plugins are responsible for scheduling the processing tasks and managing parallelism for performance.
+
+The built-in executor plugin is ``LocalExecutorPlugin`` that runs tasks using multiple threads. It has a shared thread pool and schedules tasks at most ``(number of available CPU cores) * 2`` tasks in parallel. Number of threads is configurable using ``max_threads`` system parameter.
+
+Another available executor is `embulk-executor-mapreduce <https://github.com/embulk/embulk-executor-mapreduce>`_ plugin. This executor plugin runs tasks on Hadoop, a distributed computing environment. It is suitable for processing TBs of data. An unique functionality is that it supports partitioning data by a certain column before passing them to output plugins. An example use case is that the MapReduce executor partitions data by time so that files on destination storage are partitioned for each day.
+
+Plugin API
+------------------
+
+* ``exec.LocalExecutor`` class is separated into ``exec.BulkLoader`` class for interface definition and ``exec.LocalExecutorPlugin`` for implementation. If you're application is Embulk through ``LocalExecutor`` class, you need to replace it with ``BulkLoader``.
+* ``spi.ExecAction#run`` can throw ``Exception`` and ``Exec.doWith(ExecSession, ExecAction<T>)`` throws ``ExecutionException``.
+* ``spi.Buffer`` implements ``#equals`` and ``hashCode`` methods.
+
+Plugin SPI
+------------------
+
+* Added ``spi.ExecutorPlugin`` interface.
+* Added ``Embulk::ExecutorPlugin`` class.
+
+General Changes
+------------------
+
+* If there are no input tasks, the transaction is committed successfully rather than making it failed.
+
+
+Release Date
+------------------
+2015-04-07

data/lib/embulk/executor_plugin.rb

@@ -0,0 +1,23 @@
+module Embulk
+
+  require 'embulk/data_source'
+
+  class ExecutorPlugin
+    # TODO
+
+    if Embulk.java?
+      # TODO new_java
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, ExecutorPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
+    end
+  end
+
+end

data/lib/embulk/java_plugin.rb

@@ -49,6 +49,11 @@ module Embulk
       Plugin.register_java_guess(name, java_class)
     end
 
+    def self.register_executor(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_executor(name, java_class)
+    end
+
     def self.ruby_adapter_class(java_class, ruby_base_class, ruby_module)
       Class.new(ruby_base_class) do
         const_set(:JAVA_CLASS, java_class)

data/lib/embulk/plugin.rb

@@ -13,12 +13,13 @@ module Embulk
   require 'embulk/decoder_plugin'
   require 'embulk/encoder_plugin'
   require 'embulk/guess_plugin'
+  require 'embulk/executor_plugin'
   require 'embulk/java_plugin' if Embulk.java?
 
   class PluginManager
     def initialize
       @registries = {}
-      %w[input output parser formatter decoder encoder line_filter filter guess].each do |category|
+      %w[input output parser formatter decoder encoder line_filter filter guess executor].each do |category|
         @registries[category.to_sym] = PluginRegistry.new(category, "embulk/#{category}/")
       end
     end
@@ -144,6 +145,11 @@ module Embulk
                            "org.embulk.spi.GuessPlugin" => GuessPlugin)
     end
 
+    def register_java_executor(type, klass)
+      register_java_plugin(:executor, type, klass,
+                           "org.embulk.spi.ExecutorPlugin" => ExecutorPlugin)
+    end
+
     def new_java_input(type)
       lookup(:input, type).new_java
     end
@@ -176,6 +182,10 @@ module Embulk
       lookup(:guess, type).new_java
     end
 
+    def new_java_executor(type)
+      lookup(:executor, type).new_java
+    end
+
     private
 
     # TODO lookup should fallback to Java PluginSource
@@ -222,7 +232,8 @@ module Embulk
         :register_formatter, :get_formatter, :register_java_formatter, :new_java_formatter,
         :register_decoder, :get_decoder, :register_java_decoder, :new_java_decoder,
         :register_encoder, :get_encoder, :register_java_encoder, :new_java_encoder,
-        :register_guess, :get_guess, :register_java_guess, :new_java_guess
+        :register_guess, :get_guess, :register_java_guess, :new_java_guess,
+        :register_executor, :get_executor, :register_java_executor, :new_java_executor
     end
   end
 end