embulk 0.1.0

data/lib/embulk/input_plugin.rb

@@ -0,0 +1,53 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page_builder'
+
+  class InputPlugin
+    def self.transaction(config, &control)
+      raise NotImplementedError, "InputPlugin.transaction(config, &control) must be implemented"
+    end
+
+    def self.run(task, schema, index, page_builder)
+      raise NotImplementedError, "InputPlugin#run(task, schema, index, page_builder) must be implemented"
+    end
+
+    if Embulk.java?
+      def self.java_object
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::InputPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_control)
+          config = DataSource.from_java_object(java_config)
+          next_config_hash = @ruby_class.transaction(config) do |task_source_hash,columns,processor_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
+            java_schema = Schema.new(columns).java_object
+            java_commit_reports = java_control.run(java_task_source, java_schema, processor_count)
+            java_commit_reports.map {|java_commit_report|
+              DataSource.from_java_object(java_commit_report)
+            }
+          end
+          # TODO check return type of #transaction
+          return DataSource.from_ruby_hash(next_config_hash).java_object
+        end
+
+        def run(java_task_source, java_schema, processor_index, java_output)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          page_builder = PageBuilder.new(schema, java_output)
+          commit_report_hash = @ruby_class.run(task_source, schema, processor_index, page_builder)
+          return DataSource.from_ruby_hash(commit_report_hash).java_object
+        end
+      end
+    end
+  end
+
+end

data/lib/embulk/java/bootstrap.rb

@@ -0,0 +1,12 @@
+module Embulk
+  module Java
+    require 'embulk/java/imports'
+    require 'embulk/java/time_helper'
+
+    module Injected
+      # Following constats are set by org.embulk.jruby.JRubyScriptingModule:
+      #   ModelManager
+      #   BufferAllocator
+    end
+  end
+end

data/lib/embulk/java/imports.rb

@@ -0,0 +1,26 @@
+require 'java'
+
+#
+# this file is loaded by embulk/java/bootstrap.rb
+#
+
+module Embulk::Java
+  java_import 'org.embulk.spi.Buffer'
+
+  java_import 'org.embulk.config.DataSourceImpl'
+  java_import 'org.embulk.spi.time.Timestamp'
+  java_import 'org.embulk.spi.GuessPlugin'
+  java_import 'org.embulk.spi.OutputPlugin'
+  java_import 'org.embulk.spi.InputPlugin'
+  java_import 'org.embulk.spi.TransactionalPageOutput'
+  java_import 'org.embulk.spi.PageReader'
+  java_import 'org.embulk.spi.PageBuilder'
+  java_import 'org.embulk.spi.util.LineDecoder'
+  java_import 'org.embulk.spi.util.ListFileInput'
+  java_import 'org.embulk.spi.Schema'
+  java_import 'org.embulk.spi.Column'
+  java_import 'org.embulk.spi.type.Type'
+  java_import 'org.embulk.spi.type.Types'
+
+  # TODO
+end

data/lib/embulk/java/time_helper.rb

@@ -0,0 +1,77 @@
+module Embulk
+  module Java
+    #require 'embulk/java/imports'
+    require 'time'  # Date._strptime
+
+    class TimeParserHelper
+      #include Java::JRubyTimeParserHelper
+      include org.embulk.spi.time.JRubyTimeParserHelper
+
+      class Factory
+        #include Java::JRubyTimeParserHelperFactory
+        include org.embulk.spi.time.JRubyTimeParserHelperFactory
+
+        def newInstance(format_string, year, mon, day, hour, min, sec, usec)
+          default_time = Time.utc(year, mon, day, hour, min, sec, usec)
+          TimeParserHelper.new(format_string, default_time)
+        end
+      end
+
+      def initialize(format_string, default_time)
+        @format_string = format_string
+        @default_time = default_time
+      end
+
+      # Override
+      def strptime(text)
+        hash = Date._strptime(text, @format_string)
+        unless hash
+          raise Java::TimestampParseException.new
+        end
+
+        if seconds = hash[:seconds]
+          return seconds * 1_000_000
+
+        else
+          year = hash[:year]
+          mon = hash[:mon]
+          day = hash[:mday]
+          hour = hash[:hour]
+          min = hash[:min]
+          sec = hash[:sec]
+          sec_fraction = hash[:sec_fraction]
+          zone = hash[:zone]
+          usec = hash[:sec_fraction] ? hash[:sec_fraction] * 1000000 : nil
+
+          now = @default_time
+          begin
+            break if year; year = now.year
+            break if mon; mon = now.mon
+            break if day; day = now.day
+            break if hour; hour = now.hour
+            break if min; min = now.min
+            break if sec; sec = now.sec
+            break if sec_fraction; usec = now.tv_usec
+          end until true
+
+          year ||= 1970
+          mon ||= 1
+          day ||= 1
+          hour ||= 0
+          min ||= 0
+          sec ||= 0
+          usec ||= 0
+
+          @zone = zone
+          time = Time.utc(year, mon, day, hour, min, sec, usec)
+          return time.tv_sec * 1000 + time.tv_usec / 1000
+        end
+      end
+
+      # Override
+      def getZone
+        @zone
+      end
+    end
+  end
+end

data/lib/embulk/output_plugin.rb

@@ -0,0 +1,104 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page'
+
+  class OutputPlugin
+    def self.transaction(config, schema, processor_count, &control)
+      yield(config)
+      return {}
+    end
+
+    def initialize(task, schema, index)
+      @task = task
+      @schema = schema
+      @index = index
+    end
+
+    attr_reader :task, :schema, :index
+
+    def add(page)
+      raise NotImplementedError, "OutputPlugin#add(page) must be implemented"
+    end
+
+    def finish
+    end
+
+    def close
+    end
+
+    def abort
+    end
+
+    def commit
+      {}
+    end
+
+    if Embulk.java?
+      def self.java_object
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::OutputPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_schema, processor_count, java_control)
+          config = DataSource.from_java_object(java_config)
+          schema = Schema.from_java_object(java_schema)
+          next_config_hash = @ruby_class.transaction(config, schema, processor_count) do |task_source_hash|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
+            java_commit_reports = java_control.run(java_task_source)
+            java_commit_reports.map {|java_commit_report|
+              DataSource.from_java_object(java_commit_report)
+            }
+          end
+          # TODO check return type of #transaction
+          return DataSource.from_ruby_hash(next_config_hash).java_object
+        end
+
+        def open(java_task_source, java_schema, processor_index)
+          task_source = DataSource.from_java_object(java_task_source)
+          schema = Schema.from_java_object(java_schema)
+          ruby_object = @ruby_class.new(task_source, schema, processor_index)
+          return OutputAdapter.new(ruby_object, schema)
+        end
+
+        class OutputAdapter
+          include Java::TransactionalPageOutput
+
+          def initialize(ruby_object, schema)
+            @ruby_object = ruby_object
+            @schema = schema
+          end
+
+          def add(java_page)
+            @ruby_object.add Page.new(java_page, @schema)
+          end
+
+          def finish
+            @ruby_object.finish
+          end
+
+          def close
+            @ruby_object.close
+          end
+
+          def abort
+            @ruby_object.abort
+          end
+
+          def commit
+            commit_report_hash = @ruby_object.commit
+            return DataSource.from_ruby_hash(commit_report_hash).java_object
+          end
+        end
+      end
+    end
+  end
+
+end

data/lib/embulk/page.rb

@@ -0,0 +1,28 @@
+
+module Embulk
+
+  # TODO pure-ruby page reader
+
+  class Page
+    def initialize(java_page, schema)
+      @java_page = java_page
+      @schema = schema
+    end
+
+    attr_reader :schema
+
+    def each
+      schema = @schema
+      reader = Java::PageReader.new(schema.java_object)
+      begin
+        reader.setPage(@java_page)
+        while reader.nextRecord
+          yield schema.read_record(reader)
+        end
+      ensure
+        reader.close
+      end
+    end
+  end
+
+end

data/lib/embulk/page_builder.rb

@@ -0,0 +1,22 @@
+module Embulk
+
+  class PageBuilder
+    def initialize(schema, java_page_output)
+      @page_builder = Java::PageBuilder.new(Java::Injected::BufferAllocator, schema.java_object, java_page_output)
+      @schema = schema
+    end
+
+    def add(record)
+      @schema.write_record(@page_builder, record)
+    end
+
+    def finish
+      @page_builder.finish
+    end
+
+    def close
+      @page_builder.close
+    end
+  end
+
+end

data/lib/embulk/plugin.rb

@@ -0,0 +1,152 @@
+
+module Embulk
+  require 'forwardable'
+  require 'embulk/error'
+  require 'embulk/plugin_registry'
+  require 'embulk/input_plugin'
+  require 'embulk/output_plugin'
+  #require 'embulk/parser_plugin'
+  #require 'embulk/formatter_plugin'
+  #require 'embulk/decoder_plugin'
+  #require 'embulk/encoder_plugin'
+  require 'embulk/guess_plugin'
+
+  class PluginManager
+    def initialize
+      @registries = {}
+      %w[input output parser formatter decoder encoder line_filter guess].each do |category|
+        @registries[category.to_sym] = PluginRegistry.new(category, "embulk/#{category}_")
+      end
+    end
+
+    def register_input(type, klass)
+      register_plugin(:input, type, klass, InputPlugin)
+    end
+
+    def register_output(type, klass)
+      register_plugin(:output, type, klass, OutputPlugin,
+                      "Output plugin #{klass} must extend OutputPlugin")
+    end
+
+    def register_parser(type, klass)
+      register_plugin(:parser, type, klass, ParserPlugin)
+    end
+
+    def register_formatter(type, klass)
+      register_plugin(:formatter, type, klass, FormatterPlugin)
+    end
+
+    def register_decoder(type, klass)
+      register_plugin(:decoder, type, klass, DecoderPlugin)
+    end
+
+    def register_encoder(type, klass)
+      register_plugin(:encoder, type, klass, EncoderPlugin)
+    end
+
+    def register_guess(type, klass)
+      register_plugin(:guess, type, klass, GuessPlugin,
+                     "Guess plugin #{klass} must extend GuessPlugin, LineGuessPlugin, or TextGuessPlugin class")
+    end
+
+    def get_input(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_output(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_parser(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_formatter(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_decoder(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_encoder(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def get_guess(type)
+      # TODO not implemented yet
+      lookup(:guess, type)
+    end
+
+    def new_java_input(type)
+      lookup(:input, type).java_object
+    end
+
+    def new_java_output(type)
+      lookup(:output, type).java_object
+    end
+
+    def new_java_parser(type)
+      lookup(:parser, type).java_object
+    end
+
+    def new_java_formatter(type)
+      lookup(:formatter, type).java_object
+    end
+
+    def new_java_decoder(type)
+      lookup(:decoder, type).java_object
+    end
+
+    def new_java_encoder(type)
+      lookup(:encoder, type).java_object
+    end
+
+    def new_java_guess(type)
+      lookup(:guess, type).java_object
+    end
+
+    private
+
+    # TODO lookup should fallback to Java PluginSource
+    # if not found so that ruby plugins can call java plugins.
+    # call injector.newPlugin and wrap the instance in a reverse bridge object.
+
+    def lookup(category, type)
+      @registries[category].lookup(type)
+    end
+
+    def register_plugin(category, type, klass, iface, message=nil)
+      unless klass < iface
+        message ||= "Plugin #{klass} must implement #{iface}"
+        raise message
+      end
+      @registries[category].register(type, klass)
+    end
+  end
+
+  Plugin = PluginManager.new
+
+  #module Plugin
+  #  class <<self
+  #    INSTANCE = PluginManager.new
+  #
+  #    extend Forwardable
+  #
+  #    def_delegators 'INSTANCE',
+  #      :register_input, :new_input, :new_java_input,
+  #      :register_output, :new_output, :new_java_output,
+  #      :register_parser, :new_parser, :new_java_parser,
+  #      :register_formatter, :new_formatter, :new_java_formatter,
+  #      :register_decoder, :new_decoder, :new_java_decoder,
+  #      :register_encoder, :new_encoder, :new_java_encoder,
+  #      :register_guess, :new_guess, :new_java_guess
+  #  end
+  #end
+end