embulk 0.7.0-java

data/lib/embulk/guess_plugin.rb

@@ -0,0 +1,129 @@
+module Embulk
+
+  require 'embulk/guess/schema_guess'
+
+  class GuessPlugin
+    def guess(config, sample_buffer)
+      raise NotImplementedError, "GuessPlugin#guess(config, sample_buffer) must be implemented"
+    end
+
+    if Embulk.java?
+      def self.new_java
+        JavaAdapter.new(new)
+      end
+
+      class JavaAdapter
+        include Java::GuessPlugin
+
+        def initialize(ruby_guess)
+          @ruby_guess = ruby_guess
+        end
+
+        def guess(java_config, java_sample)
+          config = DataSource.from_java(java_config)
+          sample = Buffer.from_java(java_sample)
+          config_diff_hash = @ruby_guess.guess(config, sample)
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+      end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter(java_class, GuessPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+
+        def guess(config, sample)
+          java_config = config.to_java
+          java_sample = sample.to_java
+          java_config_diff = java_object.guess(java_config, java_sample)
+          return DataSource.from_java(java_config_diff)
+        end
+      end
+    end
+  end
+
+  class TextGuessPlugin < GuessPlugin
+    def guess(config, sample)
+      if config.fetch('parser', {}).fetch('charset', nil).nil?
+        require 'embulk/guess/charset'
+        charset_guess = Guess::CharsetGuessPlugin.new
+        return charset_guess.guess(config, sample)
+      end
+
+      # TODO pure-ruby LineDecoder implementation?
+      begin
+        parser_task = config.param("parser", :hash, default: {}).load_config(Java::LineDecoder::DecoderTask)
+      rescue
+        # TODO log?
+        p $!
+        p $!.backtrace
+        return DataSource.new
+      end
+
+      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), parser_task)
+      sample_text = ''
+      while decoder.nextFile
+        first = true
+        while line = decoder.poll
+          if first
+            first = false
+          else
+            sample_text << parser_task.getNewline().getString()
+          end
+          sample_text << line
+        end
+      end
+
+      return guess_text(config, sample_text);
+    end
+
+    def guess_text(config, sample_text)
+      raise NotImplementedError, "TextGuessPlugin#guess_text(config, sample_text) must be implemented"
+    end
+  end
+
+  class LineGuessPlugin < GuessPlugin
+    def guess(config, sample)
+      if config.fetch('parser', {}).fetch('charset', nil).nil?
+        require 'embulk/guess/charset'
+        charset_guess = Guess::CharsetGuessPlugin.new
+        return charset_guess.guess(config, sample)
+      end
+
+      if config.fetch('parser', {}).fetch('newline', nil).nil?
+        require 'embulk/guess/newline'
+        newline_guess = Guess::NewlineGuessPlugin.new
+        return newline_guess.guess(config, sample)
+      end
+
+      # TODO pure-ruby LineDecoder implementation?
+      begin
+        parser_task = config.param("parser", :hash, default: {}).load_config(Java::LineDecoder::DecoderTask)
+      rescue
+        # TODO log?
+        p $!
+        p $!.backtrace
+        return DataSource.new
+      end
+
+      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), parser_task)
+      sample_lines = []
+      while decoder.nextFile
+        while line = decoder.poll
+          sample_lines << line
+        end
+        sample_lines.pop unless sample_lines.empty?  # last line can be partial
+      end
+
+      return guess_lines(config, sample_lines);
+    end
+
+    def guess_lines(config, sample_lines)
+      raise NotImplementedError, "LineGuessPlugin#guess_lines(config, sample_lines) must be implemented"
+    end
+  end
+
+end

data/lib/embulk/input_plugin.rb

@@ -0,0 +1,121 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page_builder'
+
+  class InputPlugin
+    def self.transaction(config, &control)
+      raise NotImplementedError, "InputPlugin.transaction(config, &control) must be implemented"
+    end
+
+    def self.resume(task, columns, count, &control)
+      raise NotImplementedError, "#{self}.resume(task, columns, count, &control) is not implemented. This plugin is not resumable"
+    end
+
+    def self.cleanup(task, schema, count, task_reports)
+      # do nothing by default
+    end
+
+    def self.guess(config)
+      raise NotImplementedError, "#{self}.guess(config) is not implemented. This input plugin does not support guess."
+    end
+
+    def initialize(task, schema, index, page_builder)
+      @task = task
+      @schema = schema
+      @index = index
+      @page_builder = page_builder
+      init
+    end
+
+    attr_reader :task, :schema, :index, :page_builder
+
+    def init
+    end
+
+    def run
+      raise NotImplementedError, "InputPlugin#run must be implemented"
+    end
+
+    if Embulk.java?
+      def self.new_java
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::InputPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_control)
+          config = DataSource.from_java(java_config)
+          config_diff_hash = @ruby_class.transaction(config) do |task_source_hash,columns,task_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_schema = Schema.new(columns).to_java
+            java_task_reports = java_control.run(java_task_source, java_schema, task_count)
+            java_task_reports.map {|java_task_report|
+              DataSource.from_java(java_task_report)
+            }
+          end
+          # TODO check return type of #transaction
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
+        def resume(java_task_source, java_schema, task_count, java_control)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          config_diff_hash = @ruby_class.resume(task_source, schema, task_count) do |task_source_hash,columns,task_count|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_schema = Schema.new(columns).to_java
+            java_task_reports = java_control.run(java_task_source, java_schema, task_count)
+            java_task_reports.map {|java_task_report|
+              DataSource.from_java(java_task_report)
+            }
+          end
+          # TODO check return type of #resume
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
+        def cleanup(java_task_source, java_schema, task_count, java_task_reports)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          task_reports = java_task_reports.map {|c| DataSource.from_java(c) }
+          @ruby_class.cleanup(task_source, schema, task_count, task_reports)
+          return nil
+        end
+
+        def guess(java_config)
+          config = DataSource.from_java(java_config)
+          config_diff_hash = @ruby_class.guess(config)
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
+        end
+
+        def run(java_task_source, java_schema, processor_index, java_output)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          page_builder = PageBuilder.new(schema, java_output)
+          begin
+            task_report_hash = @ruby_class.new(task_source, schema, processor_index, page_builder).run
+            return DataSource.from_ruby_hash(task_report_hash).to_java
+          ensure
+            page_builder.close
+          end
+        end
+      end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, InputPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
+    end
+  end
+
+end

data/lib/embulk/java/bootstrap.rb

@@ -0,0 +1,24 @@
+module Embulk
+  module Java
+    require 'embulk/java/imports'
+    require 'embulk/java/time_helper'
+
+    module Injected
+      # Following constats are set by org.embulk.jruby.JRubyScriptingModule:
+      #   Injector
+      #   ModelManager
+      #   BufferAllocator
+    end
+
+    def self.injector
+      Injected::Injector
+    end
+
+    require 'embulk'
+    require 'embulk/error'
+    require 'embulk/buffer'
+    require 'embulk/data_source'
+    require 'embulk/plugin'
+    require 'embulk/runner'
+  end
+end

data/lib/embulk/java/imports.rb

@@ -0,0 +1,69 @@
+require 'java'
+
+#
+# this file is loaded by embulk/java/bootstrap.rb
+#
+
+module Embulk::Java
+  module Config
+    include_package 'org.embulk.config'
+  end
+
+  module Exec
+    include_package 'org.embulk.exec'
+  end
+
+  module Plugin
+    include_package 'org.embulk.plugin'
+  end
+
+  module SPI
+    module Time
+      include_package 'org.embulk.time'
+    end
+
+    module Type
+      include_package 'org.embulk.type'
+    end
+
+    module Unit
+      include_package 'org.embulk.unit'
+    end
+
+    module Util
+      include_package 'org.embulk.util'
+    end
+
+    include_package 'org.embulk.spi'
+  end
+
+  include_package 'org.embulk'
+
+  java_import 'org.embulk.spi.Buffer'
+  java_import 'org.embulk.config.DataSourceImpl'
+  java_import 'org.embulk.spi.time.Timestamp'
+  java_import 'org.embulk.spi.time.TimestampParseException'
+  java_import 'org.embulk.spi.GuessPlugin'
+  java_import 'org.embulk.spi.OutputPlugin'
+  java_import 'org.embulk.spi.FilterPlugin'
+  java_import 'org.embulk.spi.InputPlugin'
+  java_import 'org.embulk.spi.ParserPlugin'
+  java_import 'org.embulk.spi.FormatterPlugin'
+  java_import 'org.embulk.spi.EncoderPlugin'
+  java_import 'org.embulk.spi.DecoderPlugin'
+  java_import 'org.embulk.spi.TransactionalPageOutput'
+  java_import 'org.embulk.spi.PageReader'
+  java_import 'org.embulk.spi.PageBuilder'
+  java_import 'org.embulk.spi.util.DynamicPageBuilder'
+  java_import 'org.embulk.spi.util.LineDecoder'
+  java_import 'org.embulk.spi.util.ListFileInput'
+  java_import 'org.embulk.spi.Schema'
+  java_import 'org.embulk.spi.Column'
+  java_import 'org.embulk.spi.type.Type'
+  java_import 'org.embulk.spi.type.Types'
+  java_import 'org.embulk.spi.FileInputRunner'
+  java_import 'org.embulk.spi.FileOutputRunner'
+  java_import 'org.embulk.plugin.PluginClassLoaderFactory'
+
+  # TODO
+end

data/lib/embulk/java/time_helper.rb

@@ -0,0 +1,79 @@
+module Embulk
+  module Java
+    #require 'embulk/java/imports'
+    require 'time'  # Date._strptime
+
+    class TimeParserHelper
+      #include Java::JRubyTimeParserHelper
+      include org.embulk.spi.time.JRubyTimeParserHelper
+
+      class Factory
+        #include Java::JRubyTimeParserHelperFactory
+        include org.embulk.spi.time.JRubyTimeParserHelperFactory
+
+        def newInstance(format_string, year, mon, day, hour, min, sec, usec)
+          default_time = Time.utc(year, mon, day, hour, min, sec, usec)
+          TimeParserHelper.new(format_string, default_time)
+        end
+      end
+
+      def initialize(format_string, default_time)
+        @format_string = format_string
+        @default_time = default_time
+      end
+
+      # Override
+      def strptimeUsec(text)
+        hash = Date._strptime(text, @format_string)
+        unless hash
+          raise Java::TimestampParseException.new("Failed to parse '" + text + "'")
+        end
+
+        if seconds = hash[:seconds]
+          sec_fraction = hash[:sec_fraction]  # Rational
+          usec = sec_fraction * 1_000_000 if sec_fraction
+          return seconds * 1_000_000 + usec.to_i
+
+        else
+          year = hash[:year]
+          mon = hash[:mon]
+          day = hash[:mday]
+          hour = hash[:hour]
+          min = hash[:min]
+          sec = hash[:sec]
+          sec_fraction = hash[:sec_fraction]
+          usec = sec_fraction * 1_000_000 if sec_fraction
+          zone = hash[:zone]
+
+          now = @default_time
+          begin
+            break if year; year = now.year
+            break if mon; mon = now.mon
+            break if day; day = now.day
+            break if hour; hour = now.hour
+            break if min; min = now.min
+            break if sec; sec = now.sec
+            break if sec_fraction; usec = now.tv_usec
+          end until true
+
+          year ||= 1970
+          mon ||= 1
+          day ||= 1
+          hour ||= 0
+          min ||= 0
+          sec ||= 0
+          usec ||= 0
+
+          @zone = zone
+          time = Time.utc(year, mon, day, hour, min, sec, usec)
+          return time.tv_sec * 1_000_000 + time.tv_usec
+        end
+      end
+
+      # Override
+      def getZone
+        @zone
+      end
+    end
+  end
+end

data/lib/embulk/java_plugin.rb

@@ -0,0 +1,90 @@
+module Embulk
+
+  require 'embulk/plugin'
+
+  class JavaPlugin
+    def self.classloader(dir)
+      jars = Dir["#{dir}/**/*.jar"]
+      urls = jars.map {|jar| java.io.File.new(File.expand_path(jar)).toURI.toURL }
+      factory = Java.injector.getInstance(Java::PluginClassLoaderFactory.java_class)
+      factory.create(urls, JRuby.runtime.getJRubyClassLoader())
+    end
+
+    def self.register_input(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)  # TODO handle class not found error
+      Plugin.register_java_input(name, java_class)
+    end
+
+    def self.register_output(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_output(name, java_class)
+    end
+
+    def self.register_filter(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_filter(name, java_class)
+    end
+
+    def self.register_parser(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_parser(name, java_class)
+    end
+
+    def self.register_formatter(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_formatter(name, java_class)
+    end
+
+    def self.register_decoder(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_decoder(name, java_class)
+    end
+
+    def self.register_encoder(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_encoder(name, java_class)
+    end
+
+    def self.register_guess(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_guess(name, java_class)
+    end
+
+    def self.register_executor(name, class_fqdn, jar_dir)
+      java_class = classloader(jar_dir).loadClass(class_fqdn)
+      Plugin.register_java_executor(name, java_class)
+    end
+
+    def self.ruby_adapter_class(java_class, ruby_base_class, ruby_module)
+      Class.new(ruby_base_class) do
+        const_set(:JAVA_CLASS, java_class)
+
+        include ruby_module
+        extend ruby_module::ClassMethods
+
+        unless method_defined?(:java_object)
+          def java_object
+            @java_object ||= self.class.new_java
+          end
+        end
+
+        unless (class<<self;self;end).method_defined?(:java_class)
+          def self.java_class
+            self::JAVA_CLASS
+          end
+        end
+
+        # TODO ruby_base_class already implements new_java. So
+        #      this line returns always true:
+        #unless (class<<self;self;end).method_defined?(:new_java)
+        #      but this line could return false unexpectedly if
+        #      ruby_module::ClassMethods includes other modules.
+        unless ruby_module::ClassMethods.method_defined?(:new_java)
+          def self.new_java
+            Java.injector.getInstance(java_class)
+          end
+        end
+      end
+    end
+  end
+end