embulk 0.2.1 → 0.3.0

data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java

@@ -17,5 +17,13 @@ public interface OutputPlugin
             Schema schema, int processorCount,
             OutputPlugin.Control control);
 
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control);
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports);
+
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex);
 }

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -0,0 +1,87 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.CommitReport;
+import org.embulk.config.NextConfig;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.Schema;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.FilterPlugin;
+
+public abstract class Filters
+{
+    private Filters() { }
+
+    public static List<FilterPlugin> newFilterPlugins(ExecSession exec, List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<FilterPlugin> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(exec.newPlugin(FilterPlugin.class, config.get(PluginType.class, "type")));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources, List<Schema> filterSchemas);
+    }
+
+    public static void transaction(List<FilterPlugin> plugins, List<ConfigSource> configs,
+            Schema inputSchema, Filters.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction(inputSchema);
+    }
+
+    public static PageOutput open(List<FilterPlugin> plugins, List<TaskSource> taskSources,
+            List<Schema> filterSchemas, PageOutput output)
+    {
+        PageOutput out = output;
+        int pos = 0;
+        while (pos < plugins.size()) {
+            out = plugins.get(pos).open(taskSources.get(pos), filterSchemas.get(pos), filterSchemas.get(pos + 1), out);
+            pos++;
+        }
+        return out;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<FilterPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Filters.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private final ImmutableList.Builder<Schema> filterSchemas;
+        private int pos;
+
+        RecursiveControl(List<FilterPlugin> plugins, List<ConfigSource> configs,
+                Filters.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+            this.filterSchemas = ImmutableList.builder();
+        }
+
+        public void transaction(Schema inputSchema)
+        {
+            filterSchemas.add(inputSchema);
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), inputSchema, new FilterPlugin.Control() {
+                    public void run(TaskSource taskSource, Schema outputSchema)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction(outputSchema);
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build(), filterSchemas.build());
+            }
+        }
+    }
+}

data/embulk-core/src/test/java/org/embulk/EmbulkTestRuntime.java

@@ -3,6 +3,7 @@ package org.embulk;
 import java.util.Random;
 import org.junit.runner.Description;
 import org.junit.runners.model.Statement;
+import com.google.inject.Injector;
 import com.google.inject.Binder;
 import com.google.inject.Module;
 import org.embulk.config.ConfigSource;
@@ -49,8 +50,9 @@ public class EmbulkTestRuntime
     public EmbulkTestRuntime()
     {
         super(new TestRuntimeModule());
-        ConfigSource execConfig = new DataSourceImpl(null);
-        this.exec = new ExecSession(getInjector(), execConfig);
+        Injector injector = getInjector();
+        ConfigSource execConfig = new DataSourceImpl(injector.getInstance(ModelManager.class));
+        this.exec = new ExecSession(injector, execConfig);
     }
 
     public ExecSession getExec()

data/embulk-core/src/test/java/org/embulk/spi/TestFileInputRunner.java

@@ -43,6 +43,7 @@ public class TestFileInputRunner
             this.buffers = buffers;
         }
 
+        @Override
         public NextConfig transaction(ConfigSource config,
                 FileInputPlugin.Control control)
         {
@@ -50,6 +51,21 @@ public class TestFileInputRunner
             return null;
         }
 
+        @Override
+        public NextConfig resume(TaskSource taskSource,
+                int processorCount,
+                FileInputPlugin.Control control)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void cleanup(TaskSource taskSource,
+                int processorCount,
+                List<CommitReport> successCommitReports)
+        {
+        }
+
         public TransactionalFileInput open(TaskSource taskSource,
                 int processorIndex)
         {

data/embulk-core/src/test/java/org/embulk/spi/TestFileOutputRunner.java

@@ -41,6 +41,21 @@ public class TestFileOutputRunner
             return Exec.newNextConfig();
         }
 
+        @Override
+        public NextConfig resume(TaskSource taskSource,
+                int processorCount,
+                FileOutputPlugin.Control control)
+        {
+            throw new UnsupportedOperationException();
+        }
+
+        @Override
+        public void cleanup(TaskSource taskSource,
+                int processorCount,
+                List<CommitReport> successCommitReports)
+        {
+        }
+
         @Override
         public TransactionalFileOutput open(TaskSource taskSource,
                 final int processorIndex)

data/embulk-standards/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <groupId>org.embulk</groupId>
         <artifactId>embulk-parent</artifactId>
-        <version>0.2.1-SNAPSHOT</version>
+        <version>0.3.0-SNAPSHOT</version>
     </parent>
 
     <artifactId>embulk-standards</artifactId>

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -56,12 +56,26 @@ public class LocalFileInputPlugin
         // list files recursively
         task.setFiles(listFiles(task));
 
-        // run with threads. number of processors is same with number of files
-        control.run(task.dump(), task.getFiles().size());
+        // number of processors is same with number of files
+        int processorCount = task.getFiles().size();
+        return resume(task.dump(), processorCount, control);
+    }
 
+    @Override
+    public NextConfig resume(TaskSource taskSource,
+            int processorCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, processorCount);
         return Exec.newNextConfig();
     }
 
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int processorCount,
+            List<CommitReport> successCommitReports)
+    { }
+
     public List<String> listFiles(PluginTask task)
     {
         final ImmutableList.Builder<String> builder = ImmutableList.builder();

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileOutputPlugin.java

@@ -47,11 +47,24 @@ public class LocalFileOutputPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
-        control.run(task.dump());
+        return resume(task.dump(), processorCount, control);
+    }
 
+    @Override
+    public NextConfig resume(TaskSource taskSource,
+            int processorCount,
+            FileOutputPlugin.Control control)
+    {
+        control.run(taskSource);
         return Exec.newNextConfig();
     }
 
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int processorCount,
+            List<CommitReport> successCommitReports)
+    { }
+
     @Override
     public TransactionalFileOutput open(TaskSource taskSource, final int processorIndex)
     {

data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.standards;
 
+import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.NextConfig;
@@ -18,10 +19,22 @@ public class NullOutputPlugin
             Schema schema, int processorCount,
             OutputPlugin.Control control)
     {
-        control.run(Exec.newTaskSource());
+        return resume(Exec.newTaskSource(), schema, processorCount, control);
+    }
+
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control)
+    {
+        control.run(taskSource);
         return Exec.newNextConfig();
     }
 
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    { }
+
     @Override
     public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int processorIndex)
     {

data/embulk-standards/src/main/java/org/embulk/standards/S3FileInputPlugin.java

@@ -77,13 +77,25 @@ public class S3FileInputPlugin
         task.setFiles(listFiles(task));
 
         // number of processors is same with number of files
+        int processorCount = task.getFiles().size();
+        return resume(task.dump(), processorCount, control);
+    }
 
-        // run
-        control.run(task.dump(), task.getFiles().size());
-
+    @Override
+    public NextConfig resume(TaskSource taskSource,
+            int processorCount,
+            FileInputPlugin.Control control)
+    {
+        control.run(taskSource, processorCount);
         return Exec.newNextConfig();
     }
 
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int processorCount,
+            List<CommitReport> successCommitReports)
+    { }
+
     public static AWSCredentialsProvider getCredentialsProvider(PluginTask task)
     {
         final AWSCredentials cred = new BasicAWSCredentials(

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.standards;
 
+import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.NextConfig;
@@ -30,10 +31,23 @@ public class StdoutOutputPlugin
             OutputPlugin.Control control)
     {
         final PluginTask task = config.loadConfig(PluginTask.class);
-        control.run(task.dump());
+        return resume(task.dump(), schema, processorCount, control);
+    }
+
+    @Override
+    public NextConfig resume(TaskSource taskSource,
+            Schema schema, int processorCount,
+            OutputPlugin.Control control)
+    {
+        control.run(taskSource);
         return Exec.newNextConfig();
     }
 
+    public void cleanup(TaskSource taskSource,
+            Schema schema, int processorCount,
+            List<CommitReport> successCommitReports)
+    { }
+
     @Override
     public TransactionalPageOutput open(TaskSource taskSource, final Schema schema,
             int processorIndex)

data/lib/embulk/command/embulk_run.rb

@@ -51,6 +51,21 @@ module Embulk
       op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
         options[:nextConfigOutputPath] = path
       end
+      op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
+        options[:resumeStatePath] = path
+      end
+      args = 1..1
+
+    when :cleanup
+      op.banner = "Usage: run <config.yml>"
+      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
+      end
+      op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
+        load_paths << load_path
+      end
+      op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
+        options[:resumeStatePath] = path
+      end
       args = 1..1
 
     when :preview
@@ -118,7 +133,7 @@ module Embulk
           if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
             # data is in embulk-core jar
             resource_class = org.embulk.command.Runner.java_class
-            %w[.bundle/config embulk/input_example.rb embulk/output_example.rb Gemfile Gemfile.lock].each do |file|  # TODO get file list from the jar
+            %w[.bundle/config embulk/input_example.rb embulk/filter_example.rb embulk/output_example.rb Gemfile Gemfile.lock].each do |file|  # TODO get file list from the jar
               url = resource_class.resource("/embulk/data/bundle/#{file}").to_s
               dst = File.join(path, file)
               FileUtils.mkdir_p File.dirname(dst)

data/lib/embulk/data/bundle/embulk/filter_example.rb

@@ -0,0 +1,42 @@
+module Embulk
+  module Plugin
+
+    class FilterExample < FilterPlugin
+      # filter plugin file name must be: embulk/filter_<name>.rb
+      Plugin.register_filter('example', self)
+
+      def self.transaction(config, in_schema, &control)
+        task = {
+          'key' => config.param('key', :string, default: "filter_key"),
+          'value' => config.param('value', :string, default: "filter_value")
+        }
+
+        idx = in_schema.size
+        out_columns = in_schema + [Column.new(idx, task['key'], :string)]
+
+        puts "Example filter started."
+        yield(task, out_columns)
+        puts "Example filter finished."
+      end
+
+      def initialize(task, in_schema, out_schema, page_builder)
+        super
+        @value = task['value']
+      end
+
+      def close
+      end
+
+      def add(page)
+        page.each do |record|
+          @page_builder.add(record + [@value])
+        end
+      end
+
+      def finish
+        @page_builder.finish
+      end
+    end
+
+  end
+end

data/lib/embulk/data/bundle/embulk/input_example.rb

@@ -1,44 +1,54 @@
 module Embulk
+  module Plugin
+
+    class InputExample < InputPlugin
+      # input plugin file name must be: embulk/input_<name>.rb
+      Plugin.register_input('example', self)
+
+      def self.transaction(config, &control)
+        files = ['file1', 'file2']
+        task = {
+          'files' => files,
+          'hostname' => config.param('hostname', :string, default: nil)
+        }
+
+        columns = [
+          Column.new(0, 'file', :string),
+          Column.new(1, 'hostname', :string),
+          Column.new(2, 'col0', :long),
+          Column.new(3, 'col1', :double),
+        ]
+
+        resume(task, columns, files.length, &control)
+      end
 
-  class InputExample < InputPlugin
-    # input plugin file name must be: embulk/input_<name>.rb
-    Plugin.register_input('example', self)
-
-    def self.transaction(config, &control)
-      task = {
-        'message' => config.param('message', :string, default: nil)
-      }
-      threads = config.param('threads', :integer, default: 2)
-
-      columns = [
-        Column.new(0, 'col0', :long),
-        Column.new(1, 'col1', :double),
-        Column.new(2, 'col2', :string),
-      ]
+      def self.resume(task, columns, count, &control)
+        puts "Example input started."
+        commit_reports = yield(task, columns, count)
+        puts "Example input finished. Commit reports = #{commit_reports.to_json}"
 
-      puts "Example input started."
-      commit_reports = yield(task, columns, threads)
-      puts "Example input finished. Commit reports = #{commit_reports.to_json}"
+        next_config_diff = {}
+        return next_config_diff
+      end
 
-      return {}
-    end
+      def initialize(task, schema, index, page_builder)
+        super
+        @file = task['files'][index]
+        @hostname = task['hostname']
+      end
 
-    def initialize(task, schema, index, page_builder)
-      super
-    end
+      def run
+        puts "Example input thread #{@index}..."
 
-    def run
-      puts "Example input thread #{@index}..."
+        10.times do |i|
+          @page_builder.add([@file, @hostname, i, 10.0])
+        end
+        @page_builder.finish  # don't forget to call finish :-)
 
-      10.times do |i|
-        @page_builder.add([i, 10.0, "example"])
+        commit_report = {}
+        return commit_report
       end
-      @page_builder.finish  # don't forget to call finish :-)
-
-      commit_report = {
-      }
-      return commit_report
     end
-  end
 
+  end
 end