embulk 0.7.0-java

data/lib/embulk/plugin.rb

@@ -0,0 +1,239 @@
+
+module Embulk
+  require 'forwardable'
+  require 'embulk/error'
+  require 'embulk/plugin_registry'
+  require 'embulk/input_plugin'
+  require 'embulk/file_input_plugin'
+  require 'embulk/output_plugin'
+  require 'embulk/file_output_plugin'
+  require 'embulk/filter_plugin'
+  require 'embulk/parser_plugin'
+  require 'embulk/formatter_plugin'
+  require 'embulk/decoder_plugin'
+  require 'embulk/encoder_plugin'
+  require 'embulk/guess_plugin'
+  require 'embulk/executor_plugin'
+  require 'embulk/java_plugin' if Embulk.java?
+
+  class PluginManager
+    def initialize
+      @registries = {}
+      %w[input output parser formatter decoder encoder line_filter filter guess executor].each do |category|
+        @registries[category.to_sym] = PluginRegistry.new(category, "embulk/#{category}/")
+      end
+    end
+
+    def register_input(type, klass)
+      register_plugin(:input, type, klass, InputPlugin)
+    end
+
+    def register_output(type, klass)
+      register_plugin(:output, type, klass, OutputPlugin,
+                      "Output plugin #{klass} must extend OutputPlugin")
+    end
+
+    def register_filter(type, klass)
+      register_plugin(:filter, type, klass, FilterPlugin)
+    end
+
+    def register_parser(type, klass)
+      register_plugin(:parser, type, klass, ParserPlugin)
+    end
+
+    def register_formatter(type, klass)
+      register_plugin(:formatter, type, klass, FormatterPlugin)
+    end
+
+    ## TODO DecoderPlugin JRuby API is not written by anyone yet
+    #def register_decoder(type, klass)
+    #  register_plugin(:decoder, type, klass, DecoderPlugin)
+    #end
+
+    ## TODO EncoderPlugin JRuby API is not written by anyone yet
+    #def register_encoder(type, klass)
+    #  register_plugin(:encoder, type, klass, EncoderPlugin)
+    #end
+
+    def register_guess(type, klass)
+      register_plugin(:guess, type, klass, GuessPlugin,
+                     "Guess plugin #{klass} must extend GuessPlugin, LineGuessPlugin, or TextGuessPlugin class")
+    end
+
+    # TODO InputPlugin::RubyAdapter is not written by anyone yet
+    #def get_input(type)
+    #  lookup(:input, type)
+    #end
+
+    # TODO OutputPlugin::RubyAdapter is not written by anyone yet
+    #def get_output(type)
+    #  lookup(:output, type)
+    #end
+
+    # TODO FilterPlugin::RubyAdapter is not written by anyone yet
+    #def get_filter(type)
+    #  lookup(:filter, type)
+    #end
+
+    # TODO FilterPlugin::RubyAdapter is not written by anyone yet
+    #def get_parser(type)
+    #  # TODO not implemented yet
+    #  lookup(:parser, type)
+    #end
+
+    # TODO FormatterPlugin::RubyAdapter is not written by anyone yet
+    #def get_formatter(type)
+    #  # TODO not implemented yet
+    #  lookup(:formatter, type)
+    #end
+
+    # TODO DecoderPlugin::RubyAdapter is not written by anyone yet
+    #def get_decoder(type)
+    #  # TODO not implemented yet
+    #  lookup(:decoder, type)
+    #end
+
+    # TODO EncoderPlugin::RubyAdapter is not written by anyone yet
+    #def get_encoder(type)
+    #  # TODO not implemented yet
+    #  lookup(:encoder, type)
+    #end
+
+    def get_guess(type)
+      lookup(:guess, type)
+    end
+
+    def register_java_input(type, klass)
+      register_java_plugin(:input, type, klass,
+                           "org.embulk.spi.InputPlugin" => InputPlugin,
+                           "org.embulk.spi.FileInputPlugin" => FileInputPlugin)
+    end
+
+    def register_java_output(type, klass)
+      register_java_plugin(:output, type, klass,
+                           "org.embulk.spi.OutputPlugin" => OutputPlugin,
+                           "org.embulk.spi.FileOutputPlugin" => FileOutputPlugin)
+    end
+
+    def register_java_filter(type, klass)
+      register_java_plugin(:filter, type, klass,
+                           "org.embulk.spi.FilterPlugin" => FilterPlugin)
+    end
+
+    def register_java_parser(type, klass)
+      register_java_plugin(:parser, type, klass,
+                           "org.embulk.spi.ParserPlugin" => ParserPlugin)
+    end
+
+    def register_java_formatter(type, klass)
+      register_java_plugin(:formatter, type, klass,
+                           "org.embulk.spi.FormatterPlugin" => FormatterPlugin)
+    end
+
+    def register_java_decoder(type, klass)
+      register_java_plugin(:decoder, type, klass,
+                           "org.embulk.spi.DecoderPlugin" => DecoderPlugin)
+    end
+
+    def register_java_encoder(type, klass)
+      register_java_plugin(:encoder, type, klass,
+                           "org.embulk.spi.EncoderPlugin" => EncoderPlugin)
+    end
+
+    def register_java_guess(type, klass)
+      register_java_plugin(:guess, type, klass,
+                           "org.embulk.spi.GuessPlugin" => GuessPlugin)
+    end
+
+    def register_java_executor(type, klass)
+      register_java_plugin(:executor, type, klass,
+                           "org.embulk.spi.ExecutorPlugin" => ExecutorPlugin)
+    end
+
+    def new_java_input(type)
+      lookup(:input, type).new_java
+    end
+
+    def new_java_output(type)
+      lookup(:output, type).new_java
+    end
+
+    def new_java_filter(type)
+      lookup(:filter, type).new_java
+    end
+
+    def new_java_parser(type)
+      lookup(:parser, type).new_java
+    end
+
+    def new_java_formatter(type)
+      lookup(:formatter, type).new_java
+    end
+
+    def new_java_decoder(type)
+      lookup(:decoder, type).new_java
+    end
+
+    def new_java_encoder(type)
+      lookup(:encoder, type).new_java
+    end
+
+    def new_java_guess(type)
+      lookup(:guess, type).new_java
+    end
+
+    def new_java_executor(type)
+      lookup(:executor, type).new_java
+    end
+
+    private
+
+    # TODO lookup should fallback to Java PluginSource
+    # if not found so that ruby plugins can call java plugins.
+    # call Java.injector.newPlugin and wrap the instance in a reverse bridge object.
+
+    def lookup(category, type)
+      @registries[category].lookup(type)
+    end
+
+    def register_plugin(category, type, klass, base_class, message=nil)
+      unless klass < base_class
+        message ||= "Plugin #{klass} must inherit #{base_class}"
+        raise message
+      end
+      @registries[category].register(type, klass)
+    end
+
+    def register_java_plugin(category, type, klass, iface_map, message=nil)
+      found = iface_map.find do |iface_name,ruby_base_class|
+        iface = JRuby.runtime.getJRubyClassLoader.load_class(iface_name)
+        iface.isAssignableFrom(klass)
+      end
+      unless found
+        message ||= "Java plugin #{klass} must implement #{iface_map.keys.join(' or ')}"
+        raise message
+      end
+      adapted = found.last.from_java(klass)
+      @registries[category].register(type, adapted)
+    end
+  end
+
+  module Plugin
+    class <<self
+      INSTANCE = PluginManager.new
+
+      extend Forwardable
+
+      def_delegators 'INSTANCE',
+        :register_input, :get_input, :register_java_input, :new_java_input,
+        :register_output, :get_output, :register_java_output, :new_java_output,
+        :register_filter, :get_filter, :register_java_filter, :new_java_filter,
+        :register_parser, :get_parser, :register_java_parser, :new_java_parser,
+        :register_formatter, :get_formatter, :register_java_formatter, :new_java_formatter,
+        :register_decoder, :get_decoder, :register_java_decoder, :new_java_decoder,
+        :register_encoder, :get_encoder, :register_java_encoder, :new_java_encoder,
+        :register_guess, :get_guess, :register_java_guess, :new_java_guess,
+        :register_executor, :get_executor, :register_java_executor, :new_java_executor
+    end
+  end
+end

data/lib/embulk/plugin_registry.rb

@@ -0,0 +1,96 @@
+
+module Embulk
+  require 'embulk/error'
+  require 'embulk/logger'
+
+  class PluginRegistry
+    def initialize(category, search_prefix)
+      @category = category
+      @search_prefix = search_prefix
+      @loaded_gems = {}
+      @map = {}
+    end
+
+    attr_reader :category
+
+    def register(type, value)
+      type = type.to_sym
+      @map[type] = value
+    end
+
+    def lookup(type)
+      type = type.to_sym
+      if value = @map[type]
+        return value
+      end
+      if search(type)
+        if value = @map[type]
+          return value
+        end
+        raise PluginLoadError, "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is installed but it does not correctly register plugin."
+      else
+        raise PluginLoadError, "Unknown #{@category} plugin '#{type}'. #{@search_prefix}#{type}.rb is not installed. Run 'embulk gem search -rd embulk-#{@category}' command to find plugins."
+      end
+    end
+
+    def search(type)
+      name = "#{@search_prefix}#{type}"
+      begin
+        require_and_show name
+        return true
+      rescue LoadError => e
+        # catch LoadError but don't catch ClassNotFoundException
+        raise e if e.to_s =~ /java.lang.ClassNotFoundException/
+        raise e if $LOAD_PATH.any? {|dir| File.exists? File.join(dir, "#{name}.rb") }
+      end
+
+      # search from $LOAD_PATH
+      load_path_files = $LOAD_PATH.map do |lp|
+        lpath = File.expand_path(File.join(lp, "#{name}.rb"))
+        File.exist?(lpath) ? lpath : nil
+      end
+
+      paths = load_path_files.compact.sort  # sort to prefer newer version
+      paths.each do |path|
+        require_and_show path
+        return true
+      end
+
+      # search gems
+      if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
+        specs = Gem::Specification.find_all do |spec|
+          spec.contains_requirable_file? name
+        end
+
+        # prefer newer version
+        specs = specs.sort_by {|spec| spec.version }
+        if spec = specs.last
+          spec.require_paths.each do |lib|
+            require_and_show "#{spec.full_gem_path}/#{lib}/#{name}", spec
+          end
+          return true
+        end
+      end
+
+      return false
+    end
+
+    def require_and_show(path, spec=nil)
+      require path
+      unless spec
+        name, spec = Gem.loaded_specs.find {|name,spec|
+          #spec.files.include?(path)
+          spec.contains_requirable_file?(path)
+        }
+      end
+      if spec
+        unless @loaded_gems[spec.name]
+          Embulk.logger.info "Loaded plugin #{spec.name} (#{spec.version})"
+          @loaded_gems[spec.name]
+        end
+      else
+        Embulk.logger.info "Loaded plugin #{path} from a load path"
+      end
+    end
+  end
+end

data/lib/embulk/runner.rb

@@ -0,0 +1,184 @@
+module Embulk
+
+  # Embulk.setup initializes:
+  # Runner = EmbulkRunner.new
+
+  class EmbulkRunner
+    def initialize(embed)
+      @embed = embed  # org.embulk.EmbulkEmbed
+    end
+
+    def guess(config, options={})
+      configSource = read_config(config, options)
+      output_path = options[:next_config_output_path]
+
+      check_file_writable(output_path)
+
+      configDiff = @embed.guess(configSource)
+
+      guessedConfigSource = configSource.merge(configDiff)
+      yaml = write_config(output_path, guessedConfigSource)
+      STDERR.puts yaml
+      if output_path
+        puts "Created '#{output_path}' file."
+      else
+        puts "Use -o PATH option to write the guessed config file to a file."
+      end
+
+      nil
+    end
+
+    def preview(config, options={})
+      configSource = read_config(config, options)
+      format = options[:format]
+
+      previewResult = @embed.preview(configSource)
+
+      modelManager = @embed.getModelManager
+      printer =
+        case format || "table"
+        when "table"
+          org.embulk.command.TablePreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
+        when "vertical"
+          org.embulk.command.VerticalPreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
+        else
+          raise ArgumentError, "Unknown preview output format '#{format}'. Supported formats: table, vertical"
+        end
+
+      printer.printAllPages(previewResult.getPages)
+      printer.finish
+
+      nil
+    end
+
+    def run(config, options={})
+      configSource = read_config(config, options)
+      output_path = options[:next_config_output_path]
+      resume_state_path = options[:resume_state_path]
+
+      check_file_writable(output_path)
+      check_file_writable(resume_state_path)
+
+      if resume_state_path
+        begin
+          resumeConfig = read_yaml_config_file(resume_state_path)
+          resumeConfig = nil if resumeConfig.isEmpty
+        rescue
+          # TODO log?
+          resumeConfig = nil
+        end
+      end
+
+      if resumeConfig
+        resumableResult = @embed.resumeState(configSource, resumeConfig).resume
+      elsif resume_state_path
+        resumableResult = @embed.runResumable(configSource)
+      else
+        executionResult = @embed.run(configSource)
+      end
+
+      unless executionResult
+        unless resumableResult.isSuccessful
+          Embulk.logger.info "Writing resume state to '#{resume_state_path}'"
+          write_config(resume_state_path, resumableResult.getResumeState)
+          Embulk.logger.info "Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data."
+          raise resumableResult.getCause
+        end
+        executionResult = resumableResult.getSuccessfulResult
+      end
+
+      # delete resume file
+      if resume_state_path
+        File.delete(resume_state_path) rescue nil
+      end
+
+      configDiff = executionResult.getConfigDiff
+      Embulk.logger.info("Committed.")
+      Embulk.logger.info("Next config diff: #{configDiff.toString}")
+
+      write_config(output_path, configSource.merge(configDiff))
+    end
+
+    #def resume_state(config, options={})
+    #  configSource = read_config(config, options)
+    #  Resumed.new(self, DataSource.from_java(configSource), options)
+    #end
+
+    private
+
+    def read_config(config, options={})
+      case config
+      when String
+        case config
+        when /\.yml\.liquid$/
+          require 'liquid'
+          template_params = options[:template_params] || {}
+          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params)
+        when /\.yml$/
+          @embed.newConfigLoader.fromYamlString File.read(config)
+        else
+          raise ConfigError, "Unsupported file extension. Supported file extensions are .yml and .yml.liquid: #{config}"
+        end
+
+      when Hash, DataSource
+        DataSource.new(config).to_java
+      end
+    end
+
+    def read_yaml_config_file(path)
+      @embed.newConfigLoader.fromYamlString File.read(config)
+    end
+
+    def run_liquid(source, params)
+      require 'liquid'
+      template = Liquid::Template.parse(source)
+
+      data = {
+        "env" => ENV.to_h,
+      }.merge(params)
+
+      template.render(data)
+    end
+
+    def check_file_writable(path)
+      if path
+        # Open file with append mode and do nothing.
+        # If file is not writable, it throws an exception.
+        File.open(path, "ab")
+      end
+    end
+
+    def write_config(path, modelObject)
+      yaml = dump_yaml(modelObject)
+      if path
+        File.write(path, yaml)
+      end
+      yaml
+    end
+
+    def dump_yaml(modelObject)
+      modelManager = @embed.getModelManager
+      obj = modelManager.readObject(java.lang.Object.java_class, modelManager.writeObject(modelObject))
+      return org.yaml.snakeyaml.Yaml.new.dump(obj)
+    end
+
+    #class Runnable
+    #  def initialize(runner, config, options)
+    #    @runner = runner
+    #    @config = config
+    #    @options = options
+    #  end
+    #
+    #  attr_reader :config
+    #
+    #  def preview(options={})
+    #    @runner.preview(@config, @options.merge(options))
+    #  end
+    #
+    #  def run(options={})
+    #    @runner.run(@config, @options.merge(options))
+    #  end
+    #end
+  end
+
+end