embulk 0.7.0-java

data/lib/embulk/logger.rb

@@ -0,0 +1,154 @@
+
+module Embulk
+  # this file is required before loading embulk-core.jar
+
+  require 'logger'
+
+  class Logger
+    def initialize(*args)
+      if args.length == 1
+        a = args[0]
+        if a.is_a?(Adapter)
+          @logger = a
+        elsif a.is_a?(::Logger)
+          @logger = StandardLoggerAdapter.new(a)
+        elsif Embulk.java? && (org.slf4j.Logger rescue nil) && a.is_a?(org.slf4j.Logger)
+          @logger = Slf4jAdapter.new(a)
+        else
+          @logger = StandardLoggerAdapter.new(*args)
+        end
+      else
+        @logger = StandardLoggerAdapter.new(*args)
+      end
+    end
+
+    module Adapter
+    end
+
+    def error(message=nil, &block) @logger.error(message, &block) end
+    def warn(message=nil, &block) @logger.warn(message, &block) end
+    def info(message=nil, &block) @logger.info(message, &block) end
+    def debug(message=nil, &block) @logger.debug(message, &block) end
+    def trace(message=nil, &block) @logger.trace(message, &block) end
+
+    def error?() @logger.error? end
+    def warn?() @logger.warn? end
+    def info?() @logger.info? end
+    def debug?() @logger.debug? end
+    def trace?() @logger.trace? end
+  end
+
+  class StandardLoggerAdapter < ::Logger
+    include Logger::Adapter
+
+    def initialize(*args)
+      super
+      if Embulk.java?
+        self.formatter = lambda do |severity,datetime,progname,message|
+          "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%3N %z")} [#{severity}] (#{java.lang.Thread.currentThread.name}): #{message}\n"
+        end
+      else
+        self.formatter = lambda do |severity,datetime,progname,message|
+          "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%3N %z")} [#{severity}]: #{message}\n"
+        end
+      end
+    end
+
+    def trace(message, &block)
+      debug(message, &block)
+    end
+
+    def trace?
+      debug?
+    end
+  end
+
+  class Slf4jAdapter
+    include Logger::Adapter
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def error(message, &block)
+      if block
+        if @logger.isErrorEnabled
+          @logger.error(block.call)
+        end
+      else
+        @logger.error(message)
+      end
+    end
+
+    def warn(message, &block)
+      if block
+        if @logger.isWarnEnabled
+          @logger.warn(block.call)
+        end
+      else
+        @logger.warn(message)
+      end
+    end
+
+    def info(message, &block)
+      if block
+        if @logger.isInfoEnabled
+          @logger.info(block.call)
+        end
+      else
+        @logger.info(message)
+      end
+    end
+
+    def debug(message, &block)
+      if block
+        if @logger.isDebugEnabled
+          @logger.debug(block.call)
+        end
+      else
+        @logger.debug(message)
+      end
+    end
+
+    def trace(message, &block)
+      if block
+        if @logger.isTraceEnabled
+          @logger.trace(block.call)
+        end
+      else
+        @logger.trace(message)
+      end
+    end
+
+    def fatal?
+      @logger.isErrorEnabled()
+    end
+
+    def error?
+      @logger.isErrorEnabled()
+    end
+
+    def warn?
+      @logger.isWarnEnabled()
+    end
+
+    def debug?
+      @logger.isDebugEnabled()
+    end
+
+    def trace?
+      @logger.isTraceEnabled()
+    end
+  end
+
+  def self.logger
+    @@logger
+  end
+
+  def self.logger=(logger)
+    @@logger = logger
+  end
+
+  # default logger
+  @@logger = Logger.new(STDOUT)
+end

data/lib/embulk/output_plugin.rb

@@ -0,0 +1,150 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page'
+
+  class OutputPlugin
+    def self.transaction(config, schema, task_count, &control)
+      yield(config)
+      return {}
+    end
+
+    def self.resume(task, schema, count, &control)
+      raise NotImplementedError, "#{self}.resume(task, schema, count, &control) is not implemented. This plugin is not resumable"
+    end
+
+    def self.cleanup(task, schema, count, task_reports)
+      # do nothing by default
+    end
+
+    def initialize(task, schema, index)
+      @task = task
+      @schema = schema
+      @index = index
+      init
+    end
+
+    attr_reader :task, :schema, :index
+
+    def init
+    end
+
+    def add(page)
+      raise NotImplementedError, "OutputPlugin#add(page) must be implemented"
+    end
+
+    def finish
+    end
+
+    def close
+    end
+
+    def abort
+    end
+
+    def commit
+      {}
+    end
+
+    if Embulk.java?
+      def self.new_java
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::OutputPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_schema, task_count, java_control)
+          config = DataSource.from_java(java_config)
+          schema = Schema.from_java(java_schema)
+          config_diff_hash = @ruby_class.transaction(config, schema, task_count) do |task_source_hash|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_task_reports = java_control.run(java_task_source)
+            java_task_reports.map {|java_task_report|
+              DataSource.from_java(java_task_report)
+            }
+          end
+          # TODO check return type of #transaction
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
+        def resume(java_task_source, java_schema, task_count, java_control)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          config_diff_hash = @ruby_class.resume(task_source, schema, task_count) do |task_source_hash,columns,task_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_task_reports = java_control.run(java_task_source)
+            java_task_reports.map {|java_task_report|
+              DataSource.from_java(java_task_report)
+            }
+          end
+          # TODO check return type of #resume
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
+        def cleanup(java_task_source, java_schema, task_count, java_task_reports)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          task_reports = java_task_reports.map {|c| DataSource.from_java(c) }
+          @ruby_class.cleanup(task_source, schema, task_count, task_reports)
+          return nil
+        end
+
+        def open(java_task_source, java_schema, processor_index)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          ruby_object = @ruby_class.new(task_source, schema, processor_index)
+          return OutputAdapter.new(ruby_object, schema)
+        end
+
+        class OutputAdapter
+          include Java::TransactionalPageOutput
+
+          def initialize(ruby_object, schema)
+            @ruby_object = ruby_object
+            @schema = schema
+          end
+
+          def add(java_page)
+            # TODO reuse page reader
+            @ruby_object.add Page.new(java_page, @schema)
+          end
+
+          def finish
+            @ruby_object.finish
+          end
+
+          def close
+            @ruby_object.close
+          end
+
+          def abort
+            @ruby_object.abort
+          end
+
+          def commit
+            task_report_hash = @ruby_object.commit
+            return DataSource.from_ruby_hash(task_report_hash).to_java
+          end
+        end
+      end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, OutputPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+          # TODO transaction, resume, cleanup
+        end
+        # TODO add, finish, close, abort, commit
+      end
+    end
+  end
+
+end

data/lib/embulk/page.rb

@@ -0,0 +1,30 @@
+
+module Embulk
+
+  # TODO pure-ruby page reader
+
+  class Page
+    include Enumerable
+
+    def initialize(java_page, schema)
+      @java_page = java_page
+      @schema = schema
+    end
+
+    attr_reader :schema
+
+    def each
+      schema = @schema
+      reader = Java::PageReader.new(schema.to_java)
+      begin
+        reader.setPage(@java_page)
+        while reader.nextRecord
+          yield schema.read_record(reader)
+        end
+      ensure
+        reader.close
+      end
+    end
+  end
+
+end

data/lib/embulk/page_builder.rb

@@ -0,0 +1,76 @@
+module Embulk
+
+  org.embulk.spi.util.dynamic.AbstractDynamicColumnSetter.module_eval do
+    alias_method(:set, :setRubyObject)
+  end
+
+  class PageBuilder
+    def initialize(schema, java_page_output)
+      # TODO get task as an argument
+      task = Java::SPI::Exec.newConfigSource.load_config(Java::DynamicPageBuilder::BuilderTask.java_class)
+      @page_builder = Java::DynamicPageBuilder.new(task, Java::Injected::BufferAllocator, schema.to_java, java_page_output)
+      @schema = schema
+    end
+
+    def add(record)
+      i = 0
+      m = record.size
+      while i < m
+        @page_builder.column(i).set(record[i])
+        i += 1
+      end
+      @page_builder.addRecord
+      nil
+    end
+
+    def [](index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column_or_null(index_or_column)
+      when Column
+        @page_builder.column_or_null(index_or_column.index)
+      else
+        @page_builder.column_or_null(index_or_column)
+      end
+    end
+
+    def column(index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column(index_or_column)
+      when Column
+        @page_builder.column(index_or_column.index)
+      else
+        @page_builder.lookupColumn(index_or_column)
+      end
+    end
+
+    def column_or_skip(index_or_column)
+      case index_or_column
+      when Integer
+        @page_builder.column_or_skip(index_or_column)
+      when Column
+        @page_builder.column_or_skip(index_or_column.index)
+      else
+        @page_builder.column_or_skip(index_or_column)
+      end
+    end
+
+    def add!
+      @page_builder.add_record
+    end
+
+    def flush
+      @page_builder.flush
+    end
+
+    def finish
+      @page_builder.finish
+    end
+
+    def close
+      @page_builder.close
+    end
+  end
+
+end

data/lib/embulk/parser_plugin.rb

@@ -0,0 +1,78 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page_builder'
+  require 'embulk/file_input'
+
+  class ParserPlugin
+    def self.transaction(config, &control)
+      raise NotImplementedError, "ParserPlugin.transaction(config, &control) must be implemented"
+    end
+
+    def initialize(task, schema, page_builder)
+      @task = task
+      @schema = schema
+      @page_builder = page_builder
+      init
+    end
+
+    attr_reader :task, :schema, :page_builder
+
+    def init
+    end
+
+    def run(file_input)
+      raise NotImplementedError, "ParserPlugin#run(file_input) must be implemented"
+    end
+
+    if Embulk.java?
+      def self.new_java
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::ParserPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_control)
+          config = DataSource.from_java(java_config)
+          @ruby_class.transaction(config) do |task_source_hash,columns|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_schema = Schema.new(columns).to_java
+            java_control.run(java_task_source, java_schema)
+          end
+          nil
+        end
+
+        def run(java_task_source, java_schema, java_file_input, java_output)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          file_input = FileInput.new(java_file_input)
+          page_builder = PageBuilder.new(schema, java_output)
+          begin
+            @ruby_class.new(task_source, schema, page_builder).run(file_input)
+            nil
+          ensure
+            page_builder.close
+            # FileInput is closed by FileInputRunner
+          end
+        end
+      end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, ParserPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
+    end
+  end
+
+end