embulk 0.2.1 → 0.3.0

data/lib/embulk/data/bundle/embulk/output_example.rb

@@ -1,51 +1,58 @@
 module Embulk
+  module Plugin
 
-  class OutputExample < OutputPlugin
-    # output plugin file name must be: embulk/output_<name>.rb
-    Plugin.register_output('example', self)
+    class OutputExample < OutputPlugin
+      # output plugin file name must be: embulk/output_<name>.rb
+      Plugin.register_output('example', self)
 
-    def self.transaction(config, schema, processor_count, &control)
-      task = {
-        'message' => config.param('message', :string, default: "record")
-      }
+      def self.transaction(config, schema, count, &control)
+        task = {
+          'message' => config.param('message', :string, default: "record")
+        }
 
-      puts "Example output started."
-      commit_reports = yield(task)
-      puts "Example output finished. Commit reports = #{commit_reports.to_json}"
+        resume(task, schema, count, &control)
+      end
 
-      return {}
-    end
+      def self.resume(task, schema, count, &control)
+        puts "Example output started."
+        commit_reports = yield(task)
+        puts "Example output finished. Commit reports = #{commit_reports.to_json}"
 
-    def initialize(task, schema, index)
-      puts "Example output thread #{index}..."
-      super
-      @message = task.param('message', :string)
-      @records = 0
-    end
+        next_config_diff = {}
+        return next_config_diff
+      end
 
-    def close
-    end
+      def initialize(task, schema, index)
+        puts "Example output thread #{index}..."
+        super
+        @message = task.param('message', :string)
+        @records = 0
+      end
 
-    def add(page)
-      page.each do |record|
-        hash = Hash[schema.names.zip(record)]
-        puts "#{@message}: #{hash.to_json}"
-        @records += 1
+      def close
       end
-    end
 
-    def finish
-    end
+      def add(page)
+        page.each do |record|
+          hash = Hash[schema.names.zip(record)]
+          STDOUT.write "#{@message}: #{hash.to_json}\n"
+          @records += 1
+        end
+      end
 
-    def abort
-    end
+      def finish
+      end
+
+      def abort
+      end
 
-    def commit
-      commit_report = {
-        "records" => @records
-      }
-      return commit_report
+      def commit
+        commit_report = {
+          "records" => @records
+        }
+        return commit_report
+      end
     end
-  end
 
+  end
 end

data/lib/embulk/filter_plugin.rb

@@ -0,0 +1,86 @@
+module Embulk
+
+  class FilterPlugin
+    def self.transaction(config, in_schema, &control)
+      yield(config)
+      return {}
+    end
+
+    def initialize(task, in_schema, out_schema, page_builder)
+      @task = task
+      @in_schema = in_schema
+      @out_schema = out_schema
+      @page_builder = page_builder
+    end
+
+    attr_reader :task, :in_schema, :out_schema, :page_builder
+
+    def add(page)
+      raise NotImplementedError, "FilterPlugin#add(page) must be implemented"
+    end
+
+    def finish
+    end
+
+    def close
+    end
+
+    if Embulk.java?
+      def self.java_object
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::FilterPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_in_schema, java_control)
+          config = DataSource.from_java_object(java_config)
+          in_schema = Schema.from_java_object(java_in_schema)
+          @ruby_class.transaction(config, in_schema) do |task_source_hash, out_columns|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
+            java_out_schemas = Schema.new(out_columns).java_object
+            java_control.run(java_task_source, java_out_schemas)
+          end
+          nil
+        end
+
+        def open(java_task_source, java_in_schema, java_out_schema, java_output)
+          task_source = DataSource.from_java_object(java_task_source)
+          in_schema = Schema.from_java_object(java_in_schema)
+          out_schema = Schema.from_java_object(java_out_schema)
+          page_builder = PageBuilder.new(out_schema, java_output)
+          ruby_object = @ruby_class.new(task_source, in_schema, out_schema, page_builder)
+          return OutputAdapter.new(ruby_object, in_schema, page_builder)
+        end
+
+        class OutputAdapter
+          include Java::TransactionalPageOutput
+
+          def initialize(ruby_object, in_schema, page_builder)
+            @ruby_object = ruby_object
+            @in_schema = in_schema
+            @page_builder = page_builder
+          end
+
+          def add(java_page)
+            @ruby_object.add Page.new(java_page, @in_schema)
+          end
+
+          def finish
+            @ruby_object.finish
+          end
+
+          def close
+            @ruby_object.close
+          ensure
+              @page_builder.close
+          end
+        end
+      end
+    end
+  end
+end

data/lib/embulk/input_plugin.rb

@@ -9,6 +9,14 @@ module Embulk
       raise NotImplementedError, "InputPlugin.transaction(config, &control) must be implemented"
     end
 
+    def self.resume(task, columns, count, &control)
+      raise NotImplementedError, "#{self}.resume(task, columns, count, &control) is not implemented. This plugin is not resumable"
+    end
+
+    def self.cleanup(task, schema, count, commit_reports)
+      # do nothing by default
+    end
+
     def initialize(task, schema, index, page_builder)
       @task = task
       @schema = schema
@@ -46,12 +54,39 @@ module Embulk
           return DataSource.from_ruby_hash(next_config_hash).java_object
         end
 
+        def resume(java_task_source, java_schema, processor_count, java_control)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          next_config_hash = @ruby_class.resume(task_source, schema, processor_count) do |task_source_hash,columns,processor_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
+            java_schema = Schema.new(columns).java_object
+            java_commit_reports = java_control.run(java_task_source, java_schema, processor_count)
+            java_commit_reports.map {|java_commit_report|
+              DataSource.from_java_object(java_commit_report)
+            }
+          end
+          # TODO check return type of #resume
+          return DataSource.from_ruby_hash(next_config_hash).java_object
+        end
+
+        def cleanup(java_task_source, java_schema, processor_count, java_commit_reports)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          commit_reports = java_commit_reports.map {|c| DataSource.from_java_object(c) }
+          @ruby_class.cleanup(task_source, schema, processor_count, commit_reports)
+          return nil
+        end
+
         def run(java_task_source, java_schema, processor_index, java_output)
           task_source = DataSource.from_java_object(java_task_source)
           schema = Schema.from_java_object(java_schema)
           page_builder = PageBuilder.new(schema, java_output)
-          commit_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
-          return DataSource.from_ruby_hash(commit_report_hash).java_object
+          begin
+            commit_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
+            return DataSource.from_ruby_hash(commit_report_hash).java_object
+          ensure
+            page_builder.close
+          end
         end
       end
     end

data/lib/embulk/java/imports.rb

@@ -11,6 +11,7 @@ module Embulk::Java
   java_import 'org.embulk.spi.time.Timestamp'
   java_import 'org.embulk.spi.GuessPlugin'
   java_import 'org.embulk.spi.OutputPlugin'
+  java_import 'org.embulk.spi.FilterPlugin'
   java_import 'org.embulk.spi.InputPlugin'
   java_import 'org.embulk.spi.TransactionalPageOutput'
   java_import 'org.embulk.spi.PageReader'

data/lib/embulk/output_plugin.rb

@@ -10,6 +10,14 @@ module Embulk
       return {}
     end
 
+    def self.resume(task, schema, count, &control)
+      raise NotImplementedError, "#{self}.resume(task, schema, count, &control) is not implemented. This plugin is not resumable"
+    end
+
+    def self.cleanup(task, schema, count, commit_reports)
+      # do nothing by default
+    end
+
     def initialize(task, schema, index)
       @task = task
       @schema = schema
@@ -61,6 +69,28 @@ module Embulk
           return DataSource.from_ruby_hash(next_config_hash).java_object
         end
 
+        def resume(java_task_source, java_schema, processor_count, java_control)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          next_config_hash = @ruby_class.resume(task_source, schema, processor_count) do |task_source_hash,columns,processor_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
+            java_commit_reports = java_control.run(java_task_source)
+            java_commit_reports.map {|java_commit_report|
+              DataSource.from_java_object(java_commit_report)
+            }
+          end
+          # TODO check return type of #resume
+          return DataSource.from_ruby_hash(next_config_hash).java_object
+        end
+
+        def cleanup(java_task_source, java_schema, processor_count, java_commit_reports)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          commit_reports = java_commit_reports.map {|c| DataSource.from_java_object(c) }
+          @ruby_class.cleanup(task_source, schema, processor_count, commit_reports)
+          return nil
+        end
+
         def open(java_task_source, java_schema, processor_index)
           task_source = DataSource.from_java_object(java_task_source)
           schema = Schema.from_java_object(java_schema)

data/lib/embulk/plugin.rb

@@ -4,6 +4,7 @@ module Embulk
   require 'embulk/error'
   require 'embulk/plugin_registry'
   require 'embulk/input_plugin'
+  require 'embulk/filter_plugin'
   require 'embulk/output_plugin'
   #require 'embulk/parser_plugin'
   #require 'embulk/formatter_plugin'
@@ -14,7 +15,7 @@ module Embulk
   class PluginManager
     def initialize
       @registries = {}
-      %w[input output parser formatter decoder encoder line_filter guess].each do |category|
+      %w[input output parser formatter decoder encoder line_filter filter guess].each do |category|
         @registries[category.to_sym] = PluginRegistry.new(category, "embulk/#{category}_")
       end
     end
@@ -44,6 +45,10 @@ module Embulk
       register_plugin(:encoder, type, klass, EncoderPlugin)
     end
 
+    def register_filter(type, klass)
+      register_plugin(:filter, type, klass, FilterPlugin)
+    end
+
     def register_guess(type, klass)
       register_plugin(:guess, type, klass, GuessPlugin,
                      "Guess plugin #{klass} must extend GuessPlugin, LineGuessPlugin, or TextGuessPlugin class")
@@ -79,6 +84,11 @@ module Embulk
       lookup(:guess, type)
     end
 
+    def get_filter(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
     def get_guess(type)
       # TODO not implemented yet
       lookup(:guess, type)
@@ -108,6 +118,10 @@ module Embulk
       lookup(:encoder, type).java_object
     end
 
+    def new_java_filter(type)
+      lookup(:filter, type).java_object
+    end
+
     def new_java_guess(type)
       lookup(:guess, type).java_object
     end
@@ -131,22 +145,21 @@ module Embulk
     end
   end
 
-  Plugin = PluginManager.new
-
-  #module Plugin
-  #  class <<self
-  #    INSTANCE = PluginManager.new
-  #
-  #    extend Forwardable
-  #
-  #    def_delegators 'INSTANCE',
-  #      :register_input, :new_input, :new_java_input,
-  #      :register_output, :new_output, :new_java_output,
-  #      :register_parser, :new_parser, :new_java_parser,
-  #      :register_formatter, :new_formatter, :new_java_formatter,
-  #      :register_decoder, :new_decoder, :new_java_decoder,
-  #      :register_encoder, :new_encoder, :new_java_encoder,
-  #      :register_guess, :new_guess, :new_java_guess
-  #  end
-  #end
+  module Plugin
+    class <<self
+      INSTANCE = PluginManager.new
+
+      extend Forwardable
+
+      def_delegators 'INSTANCE',
+        :register_input, :get_input, :new_java_input,
+        :register_output, :get_output, :new_java_output,
+        :register_parser, :get_parser, :new_java_parser,
+        :register_formatter, :get_formatter, :new_java_formatter,
+        :register_decoder, :get_decoder, :new_java_decoder,
+        :register_encoder, :get_encoder, :new_java_encoder,
+        :register_filter, :get_filter, :new_java_filter,
+        :register_guess, :get_guess, :new_java_guess
+    end
+  end
 end

data/lib/embulk/schema.rb

@@ -6,24 +6,30 @@ module Embulk
     def initialize(src)
       super
 
-      record_reader_script = "lambda do |reader|\n"
-      record_reader_script << "record = []\n"
+      record_reader_script =
+        "lambda do |reader|\n" <<
+        "record = []\n"
       each do |column|
+        idx = column.index
         column_script =
+          "if reader.isNull(#{idx})\n" <<
+          "record << nil\n" <<
+          "else\n" <<
           case column.type
           when :boolean
-            "record << reader.getBoolean(#{column.index})"
+            "record << reader.getBoolean(#{idx})"
           when :long
-            "record << reader.getLong(#{column.index})"
+            "record << reader.getLong(#{idx})"
           when :double
-            "record << reader.getDouble(#{column.index})"
+            "record << reader.getDouble(#{idx})"
           when :string
-            "record << reader.getString(#{column.index})"
+            "record << reader.getString(#{idx})"
           when :timestamp
-            "record << reader.getTimestamp(#{column.index}).getRubyTime(JRuby.runtime)"
+            "record << reader.getTimestamp(#{idx}).getRubyTime(JRuby.runtime)"
           else
             raise "Unknown type #{column.type.inspect}"
-          end
+          end <<
+          "end\n"
         record_reader_script << column_script << "\n"
       end
       record_reader_script << "record\n"
@@ -34,7 +40,8 @@ module Embulk
       record_writer_script << "java_timestamp_class = ::Embulk::Java::Timestamp\n"
       each do |column|
         idx = column.index
-        column_script = "if record[#{idx}].nil?\n" <<
+        column_script =
+          "if record[#{idx}].nil?\n" <<
           "builder.setNull(#{idx})\n" <<
           "else\n" <<
           case column.type

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = "0.2.1"
+  VERSION = "0.3.0"
 end

data/pom.xml

@@ -4,7 +4,7 @@
 
     <groupId>org.embulk</groupId>
     <artifactId>embulk-parent</artifactId>
-    <version>0.2.1-SNAPSHOT</version>
+    <version>0.3.0-SNAPSHOT</version>
     <packaging>pom</packaging>
 
     <name>Embulk</name>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-29 00:00:00.000000000 Z
+date: 2015-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -122,9 +122,9 @@ files:
 - classpath/commons-codec-1.3.jar
 - classpath/commons-lang3-3.1.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-cli-0.2.1-SNAPSHOT.jar
-- classpath/embulk-core-0.2.1-SNAPSHOT.jar
-- classpath/embulk-standards-0.2.1-SNAPSHOT.jar
+- classpath/embulk-cli-0.3.0-SNAPSHOT.jar
+- classpath/embulk-core-0.3.0-SNAPSHOT.jar
+- classpath/embulk-standards-0.3.0-SNAPSHOT.jar
 - classpath/guava-17.0.jar
 - classpath/guice-3.0.jar
 - classpath/guice-multibindings-3.0.jar
@@ -177,18 +177,20 @@ files:
 - embulk-core/src/main/java/org/embulk/config/TaskValidationException.java
 - embulk-core/src/main/java/org/embulk/config/TaskValidator.java
 - embulk-core/src/main/java/org/embulk/exec/ExecModule.java
-- embulk-core/src/main/java/org/embulk/exec/ExecuteInterruptedException.java
-- embulk-core/src/main/java/org/embulk/exec/ExecuteResult.java
+- embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java
+- embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java
 - embulk-core/src/main/java/org/embulk/exec/ExtensionServiceLoaderModule.java
 - embulk-core/src/main/java/org/embulk/exec/ForSystemConfig.java
 - embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java
 - embulk-core/src/main/java/org/embulk/exec/LocalExecutor.java
 - embulk-core/src/main/java/org/embulk/exec/LoggerProvider.java
 - embulk-core/src/main/java/org/embulk/exec/NoSampleException.java
+- embulk-core/src/main/java/org/embulk/exec/PartialExecutionException.java
 - embulk-core/src/main/java/org/embulk/exec/PooledBufferAllocator.java
 - embulk-core/src/main/java/org/embulk/exec/PreviewExecutor.java
 - embulk-core/src/main/java/org/embulk/exec/PreviewResult.java
 - embulk-core/src/main/java/org/embulk/exec/PreviewedNoticeError.java
+- embulk-core/src/main/java/org/embulk/exec/ResumeState.java
 - embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java
 - embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java
 - embulk-core/src/main/java/org/embulk/jruby/JRubyPluginSource.java
@@ -216,6 +218,7 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/FileOutput.java
 - embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java
 - embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java
+- embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java
 - embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java
 - embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java
 - embulk-core/src/main/java/org/embulk/spi/InputPlugin.java
@@ -256,6 +259,7 @@ files:
 - embulk-core/src/main/java/org/embulk/spi/util/Encoders.java
 - embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java
 - embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java
+- embulk-core/src/main/java/org/embulk/spi/util/Filters.java
 - embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java
 - embulk-core/src/main/java/org/embulk/spi/util/Inputs.java
 - embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java
@@ -320,10 +324,12 @@ files:
 - lib/embulk/data/bundle/.bundle/config
 - lib/embulk/data/bundle/Gemfile
 - lib/embulk/data/bundle/Gemfile.lock
+- lib/embulk/data/bundle/embulk/filter_example.rb
 - lib/embulk/data/bundle/embulk/input_example.rb
 - lib/embulk/data/bundle/embulk/output_example.rb
 - lib/embulk/data_source.rb
 - lib/embulk/error.rb
+- lib/embulk/filter_plugin.rb
 - lib/embulk/guess_charset.rb
 - lib/embulk/guess_csv.rb
 - lib/embulk/guess_gzip.rb