embulk 0.3.2 → 0.4.0

data/lib/embulk/data/package_data.rb

@@ -0,0 +1,64 @@
+module Embulk
+
+  class PackageData
+    if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
+      # data is in embulk-core jar
+      resource_class = org.embulk.command.Runner.java_class
+      JAVA_RESOURCE = true
+      RESOURCE_URL = resource_class.resource("/embulk/data")
+    else
+      JAVA_RESOURCE = false
+      FILE_BASE_PATH = File.join(Embulk.home('lib'), 'embulk', 'data')
+    end
+
+    def initialize(base_name, dest_dir, erb_binding=nil)
+      require 'fileutils'
+      @base_name = base_name
+      @dest_dir = dest_dir
+      @erb_binding = erb_binding
+    end
+
+    def path(src)
+      if JAVA_RESOURCE
+        "#{RESOURCE_URL}/#{@base_name}/#{src}"
+      else
+        File.join(FILE_BASE_PATH, @base_name, src)
+      end
+    end
+
+    def content(src)
+      File.read(path(src))
+    end
+
+    def erb(src)
+      require 'erb'
+      ERB.new(content(src), nil, '%').result(@erb_binding)
+    end
+
+    def cp(src, dest_name)
+      path = dest_path_message(dest_name)
+      FileUtils.cp path(src), path
+    end
+
+    def cp_erb(src, dest_name)
+      path = dest_path_message(dest_name)
+      File.open(path, "w") {|f| f.write erb(src) }
+    end
+
+    def dest_path(dest_name)
+      File.join(@dest_dir, *dest_name.split('/'))
+    end
+
+    def dest_path_message(dest_name)
+      path = dest_path(dest_name)
+      puts "  Creating #{path}"
+      FileUtils.mkdir_p File.dirname(path)
+      path
+    end
+
+    def set_executable(dest_name)
+      File.chmod(0755, dest_path(dest_name))
+    end
+  end
+
+end

data/lib/embulk/data_source.rb

@@ -39,7 +39,7 @@ module Embulk
     end
 
     if Embulk.java?
-      def self.from_java_object(java_data_source_impl)
+      def self.from_java(java_data_source_impl)
         json = java_data_source_impl.toString
         new.merge!(JSON.parse(json))
       end
@@ -48,7 +48,7 @@ module Embulk
         new.merge!(hash)
       end
 
-      def java_object
+      def to_java
         json = to_json
         Java::Injected::ModelManager.readObject(Java::DataSourceImpl.java_class, json.to_java)
       end

data/lib/embulk/decoder_plugin.rb

@@ -0,0 +1,27 @@
+module Embulk
+
+  require 'embulk/data_source'
+
+  class DecoderPlugin
+    def self.transaction(config, &control)
+      raise NotImplementedError, "DecoderPlugin.transaction(config, &control) must be implemented"
+    end
+
+    # TODO
+
+    if Embulk.java?
+      # TODO new_java
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, DecoderPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
+    end
+  end
+
+end

data/lib/embulk/encoder_plugin.rb

@@ -0,0 +1,27 @@
+module Embulk
+
+  require 'embulk/data_source'
+
+  class EncoderPlugin
+    def self.transaction(config, &control)
+      raise NotImplementedError, "EncoderPlugin.transaction(config, &control) must be implemented"
+    end
+
+    # TODO
+
+    if Embulk.java?
+      # TODO new_java
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, EncoderPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
+    end
+  end
+
+end

data/lib/embulk/error.rb

@@ -2,4 +2,7 @@
 module Embulk
   class ConfigError < StandardError
   end
+
+  class PluginLoadError < StandardError
+  end
 end

data/lib/embulk/file_input_plugin.rb

@@ -0,0 +1,27 @@
+module Embulk
+
+  class FileInputPlugin
+    # TODO transaction, resume, cleanup
+    # TODO run
+
+    if Embulk.java?
+      # TODO to_java
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, FileInputPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+          def new_java
+            Java::FileInputRunner.new(Java.injector.getInstance(java_class))
+          end
+          # TODO transaction, resume, cleanup
+        end
+
+        # TODO run
+      end
+    end
+  end
+
+end

data/lib/embulk/file_output_plugin.rb

@@ -0,0 +1,27 @@
+module Embulk
+
+  class FileOutputPlugin
+    # TODO transaction, resume, cleanup
+    # TODO add, finish, close, abort, commit
+
+    if Embulk.java?
+      # TODO to_java
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, FileOutputPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+          def new_java
+            Java::FileOutputRunner.new(Java.injector.getInstance(java_class))
+          end
+          # TODO transaction, resume, cleanup
+        end
+
+        # TODO add, finish, close, abort, commit
+      end
+    end
+  end
+
+end

data/lib/embulk/filter_plugin.rb

@@ -1,5 +1,10 @@
 module Embulk
 
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page'
+  require 'embulk/page_builder'
+
   class FilterPlugin
     def self.transaction(config, in_schema, &control)
       yield(config)
@@ -11,10 +16,14 @@ module Embulk
       @in_schema = in_schema
       @out_schema = out_schema
       @page_builder = page_builder
+      init
     end
 
     attr_reader :task, :in_schema, :out_schema, :page_builder
 
+    def init
+    end
+
     def add(page)
       raise NotImplementedError, "FilterPlugin#add(page) must be implemented"
     end
@@ -26,7 +35,7 @@ module Embulk
     end
 
     if Embulk.java?
-      def self.java_object
+      def self.new_java
         JavaAdapter.new(self)
       end
 
@@ -38,20 +47,20 @@ module Embulk
         end
 
         def transaction(java_config, java_in_schema, java_control)
-          config = DataSource.from_java_object(java_config)
-          in_schema = Schema.from_java_object(java_in_schema)
+          config = DataSource.from_java(java_config)
+          in_schema = Schema.from_java(java_in_schema)
           @ruby_class.transaction(config, in_schema) do |task_source_hash, out_columns|
-            java_task_source = DataSource.from_ruby_hash(task_source_hash).java_object
-            java_out_schemas = Schema.new(out_columns).java_object
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_out_schemas = Schema.new(out_columns).to_java
             java_control.run(java_task_source, java_out_schemas)
           end
           nil
         end
 
         def open(java_task_source, java_in_schema, java_out_schema, java_output)
-          task_source = DataSource.from_java_object(java_task_source)
-          in_schema = Schema.from_java_object(java_in_schema)
-          out_schema = Schema.from_java_object(java_out_schema)
+          task_source = DataSource.from_java(java_task_source)
+          in_schema = Schema.from_java(java_in_schema)
+          out_schema = Schema.from_java(java_out_schema)
           page_builder = PageBuilder.new(out_schema, java_output)
           ruby_object = @ruby_class.new(task_source, in_schema, out_schema, page_builder)
           return OutputAdapter.new(ruby_object, in_schema, page_builder)
@@ -77,10 +86,20 @@ module Embulk
           def close
             @ruby_object.close
           ensure
-              @page_builder.close
+            @page_builder.close
           end
         end
       end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, FilterPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+        end
+        # TODO
+      end
     end
   end
 end

data/lib/embulk/formatter_plugin.rb

@@ -0,0 +1,105 @@
+module Embulk
+
+  require 'embulk/data_source'
+  require 'embulk/schema'
+  require 'embulk/page'
+  #require 'embulk/file_output'  TODO not implemented
+
+  class FormatterPlugin
+    def self.transaction(config, schema, &control)
+      yield(config)
+      return {}
+    end
+
+    def initialize(task, schema, file_output)
+      @task = task
+      @schema = schema
+      @file_output
+      init
+    end
+
+    attr_reader :task, :schema, :file_output
+
+    def init
+    end
+
+    def add(page)
+      raise NotImplementedError, "FormatterPlugin#add(page) must be implemented"
+    end
+
+    def finish
+    end
+
+    def close
+    end
+
+    if Embulk.java?
+      def self.new_java
+        JavaAdapter.new(self)
+      end
+
+      class JavaAdapter
+        include Java::FormatterPlugin
+
+        def initialize(ruby_class)
+          @ruby_class = ruby_class
+        end
+
+        def transaction(java_config, java_schema, java_control)
+          config = DataSource.from_java(java_config)
+          schema = Schema.from_java(java_schema)
+          @ruby_class.transaction(config, schema) do |task_source_hash|
+            java_task_source = DataSource.from_ruby_hash(task_source_hash).to_java
+            java_control.run(java_task_source)
+          end
+          nil
+        end
+
+        def open(java_task_source, java_schema, java_file_output)
+          task_source = DataSource.from_java(java_task_source)
+          schema = Schema.from_java(java_schema)
+          file_output = FileOutput.from_java(java_file_output)
+          ruby_object = @ruby_class.new(task_source, schema, file_output)
+          return OutputAdapter.new(ruby_object, schema, file_output)
+        end
+
+        class OutputAdapter
+          include Java::TransactionalPageOutput
+
+          def initialize(ruby_object, schema, file_output)
+            @ruby_object = ruby_object
+            @schema = schema
+            @file_output = file_output
+          end
+
+          def add(java_page)
+            # TODO reuse page reader
+            @ruby_object.add Page.new(java_page, @schema)
+          end
+
+          def finish
+            @ruby_object.finish
+          end
+
+          def close
+            @ruby_object.close
+          ensure
+            @file_output.close
+          end
+        end
+      end
+
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter_class(java_class, FormatterPlugin, RubyAdapter)
+      end
+
+      module RubyAdapter
+        module ClassMethods
+          # TODO transaction, resume, cleanup
+        end
+        # TODO add, finish, close
+      end
+    end
+  end
+
+end

data/lib/embulk/guess_csv.rb

@@ -12,6 +12,15 @@ module Embulk
       "\"", "'"
     ]
 
+    # CsvParserPlugin.TRUE_STRINGS
+    TRUE_STRINGS = Hash[*%w[
+      true True TRUE
+      yes Yes YES
+      y Y
+      on On ON
+      1
+    ].map {|k| [k, true] }]
+
     def guess_lines(config, sample_lines)
       delim = guess_delimiter(sample_lines)
       unless delim
@@ -152,7 +161,7 @@ module Embulk
     end
 
     def guess_type(str)
-      if ["true", "false"].include?(str)
+      if TRUE_STRINGS[str]
         return "boolean"
       end
 

data/lib/embulk/guess_plugin.rb

@@ -6,43 +6,38 @@ module Embulk
     end
 
     if Embulk.java?
-      def self.java_object
+      def self.new_java
         JavaAdapter.new(new)
       end
 
-      def self.from_java_object(java_guess)
-        RubyAdapter.new(java_guess)
-      end
-
-      class RubyAdapter < Embulk::GuessPlugin
-        def initialized(java_guess)
-          @java_guess = java_guess
-        end
+      class JavaAdapter
+        include Java::GuessPlugin
 
-        def guess(config, sample)
-          java_config = config.java_object
-          java_sample = sample.java_object
-          java_next_config = @java_guess.guess(java_config, java_sample)
-          return DataSource.from_java_object(java_next_config)
+        def initialize(ruby_guess)
+          @ruby_guess = ruby_guess
         end
 
-        def java_object
-          @java_guess
+        def guess(java_config, java_sample)
+          config = DataSource.from_java(java_config)
+          sample = Buffer.from_java(java_sample)
+          config_diff_hash = @ruby_guess.guess(config, sample)
+          return DataSource.from_ruby_hash(config_diff_hash).to_java
         end
       end
 
-      class JavaAdapter
-        include Java::GuessPlugin
+      def self.from_java(java_class)
+        JavaPlugin.ruby_adapter(java_class, GuessPlugin, RubyAdapter)
+      end
 
-        def initialize(ruby_guess)
-          @ruby_guess = ruby_guess
+      module RubyAdapter
+        module ClassMethods
         end
 
-        def guess(java_config, java_sample)
-          config = DataSource.from_java_object(java_config)
-          sample = Buffer.from_java_object(java_sample)
-          next_config_hash = @ruby_guess.guess(config, sample)
-          return DataSource.from_ruby_hash(next_config_hash).java_object
+        def guess(config, sample)
+          java_config = config.to_java
+          java_sample = sample.to_java
+          java_config_diff = java_object.guess(java_config, java_sample)
+          return DataSource.from_java(java_config_diff)
         end
       end
     end
@@ -60,7 +55,7 @@ module Embulk
         return DataSource.new
       end
 
-      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.java_object]]), task)
+      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), task)
       sample_text = ''
       while decoder.nextFile
         first = true
@@ -94,7 +89,7 @@ module Embulk
         return DataSource.new
       end
 
-      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.java_object]]), task)
+      decoder = Java::LineDecoder.new(Java::ListFileInput.new([[sample.to_java]]), task)
       sample_lines = []
       while decoder.nextFile
         while line = decoder.poll