embulk 0.6.27 → 0.7.0

data/embulk-standards/src/main/java/org/embulk/standards/NullOutputPlugin.java

@@ -4,7 +4,7 @@ import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Page;
 import org.embulk.spi.Exec;
@@ -32,7 +32,7 @@ public class NullOutputPlugin
 
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successTaskReports)
     { }
 
     @Override
@@ -50,9 +50,9 @@ public class NullOutputPlugin
 
             public void abort() { }
 
-            public CommitReport commit()
+            public TaskReport commit()
             {
-                return Exec.newCommitReport();
+                return Exec.newTaskReport();
             }
         };
     }

data/embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java

@@ -3,8 +3,6 @@ package org.embulk.standards;
 import com.google.common.base.Preconditions;
 import com.google.inject.Binder;
 import com.google.inject.Module;
-import com.google.inject.name.Names;
-import com.google.inject.multibindings.Multibinder;
 import org.embulk.spi.FilterPlugin;
 import org.embulk.spi.FormatterPlugin;
 import org.embulk.spi.InputPlugin;
@@ -12,7 +10,6 @@ import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.DecoderPlugin;
 import org.embulk.spi.EncoderPlugin;
-import org.embulk.exec.GuessExecutor;
 import org.embulk.plugin.PluginType;
 import static org.embulk.plugin.InjectedPluginSource.registerPluginTo;
 import static org.embulk.exec.GuessExecutor.registerDefaultGuessPluginTo;

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -4,12 +4,10 @@ import java.util.List;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.Task;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.Schema;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.Column;
 import org.embulk.spi.Page;
 import org.embulk.spi.Exec;
 import org.embulk.spi.OutputPlugin;
@@ -45,7 +43,7 @@ public class StdoutOutputPlugin
 
     public void cleanup(TaskSource taskSource,
             Schema schema, int taskCount,
-            List<CommitReport> successCommitReports)
+            List<TaskReport> successTaskReports)
     { }
 
     @Override
@@ -78,9 +76,9 @@ public class StdoutOutputPlugin
 
             public void abort() { }
 
-            public CommitReport commit()
+            public TaskReport commit()
             {
-                return Exec.newCommitReport();
+                return Exec.newTaskReport();
             }
         };
     }

data/embulk.gemspec

@@ -12,12 +12,25 @@ Gem::Specification.new do |gem|
   gem.license       = "Apache 2.0"
   gem.homepage      = "https://github.com/embulk/embulk"
 
-  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
+  provided_classpath = Dir["classpath/jruby-complete-*.jar"] + Dir["classpath/icu4j-*.jar"]
+  gem.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"] - provided_classpath
   gem.test_files    = gem.files.grep(%r"^(test|spec)/")
   gem.executables   = gem.files.grep(%r"^bin/").map{ |f| File.basename(f) }
   gem.require_paths = ["lib"]
   gem.has_rdoc      = false
 
+  if RUBY_PLATFORM =~ /java/i
+    gem.add_dependency "liquid", '~> 3.0.6'
+
+    # For embulk/guess/charset.rb. See also embulk-core/build.gradle
+    gem.add_dependency "rjack-icu", '~> 4.54.1.1'
+
+    gem.platform = 'java'
+
+  else
+    gem.add_dependency "jruby-jars", '= 9.0.0.0'
+  end
+
   gem.add_development_dependency "bundler", [">= 1.0"]
   gem.add_development_dependency "rake", [">= 0.10.0"]
   gem.add_development_dependency "test-unit", ["~> 3.0.9"]

data/lib/embulk.rb

@@ -9,9 +9,64 @@ module Embulk
     end
   end
 
+  # logger should be setup first
   require 'embulk/logger'
-  require 'embulk/error'
-  require 'embulk/buffer'
-  require 'embulk/data_source'
-  require 'embulk/plugin'
+
+  def self.lib_path(path)
+    path = '' if path == '/'
+    jar, resource = __FILE__.split("!", 2)
+    if resource
+      lib = resource.split("/")[0..-2].join("/")
+      "#{jar}!#{lib}/#{path}"
+    elsif __FILE__ =~ /^classpath:/
+      lib = __FILE__.split("/")[0..-2].join("/")
+      "#{lib}/#{path}"
+    else
+      lib = File.expand_path File.dirname(__FILE__)
+      File.join(lib, *path.split("/"))
+    end
+  end
+
+  def self.require_classpath
+    if __FILE__.include?("!")
+      # single jar
+      jar, resource = __FILE__.split("!", 2)
+      require jar
+
+    elsif __FILE__ =~ /^classpath:/
+      # already in classpath
+
+    else
+      # gem package
+      gem_root = File.expand_path('..', File.dirname(__FILE__))
+      classpath_dir = File.join(gem_root, "classpath")
+      jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
+      jars.each do |jar|
+        require File.join(classpath_dir, jar)
+      end
+    end
+  end
+
+  def self.setup(system_config={})
+    unless RUBY_PLATFORM =~ /java/i
+      raise "Embulk.setup works only with JRuby."
+    end
+
+    require_classpath
+
+    systemConfigJson = system_config.merge({
+      # use the global ruby runtime for all ScriptingContainer
+      # injected by org.embulk.jruby.JRubyScriptingModule
+      use_global_ruby_runtime: true
+    }).to_json
+
+    bootstrap = org.embulk.EmbulkEmbed::Bootstrap.new
+    systemConfig = bootstrap.getSystemConfigLoader.fromJsonString(systemConfigJson)
+    bootstrap.setSystemConfig(systemConfig)
+    embed = bootstrap.java_method(:initialize).call  # see embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.
+
+    # see also embulk/java/bootstrap.rb loaded by JRubyScriptingModule
+
+    Embulk.const_set :Runner, EmbulkRunner.new(embed)
+  end
 end

data/lib/embulk/command/embulk_bundle.rb

@@ -0,0 +1,56 @@
+
+bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
+bundle_path = nil if bundle_path.empty?
+
+# search -b, --bundle BUNDLE_DIR
+bundle_path_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+if bundle_path_index
+  bundle_path = ARGV.slice!(bundle_path_index, 2)[1]
+end
+
+if bundle_path
+  # use bundler installed at bundle_path
+  ENV['EMBULK_BUNDLE_PATH'] = bundle_path
+  ENV['GEM_HOME'] = File.expand_path File.join(bundle_path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  ENV['GEM_PATH'] = ''
+  Gem.clear_paths  # force rubygems to reload GEM_HOME
+
+  ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
+
+  begin
+    require 'bundler'
+  rescue LoadError => e
+    raise "#{e}\nBundler is not installed. Did you run \`$ embulk bundle #{bundle_path}\` ?"
+  end
+  Bundler.load.setup_environment
+  require 'bundler/setup'
+  # since here, `require` may load files of different (newer) embulk versions
+  # especially following 'embulk/command/embulk_main'.
+
+  # add bundle directory path to load local plugins at ./embulk
+  $LOAD_PATH << File.expand_path(bundle_path)
+
+  begin
+    require 'embulk/command/embulk_main'
+  rescue LoadError
+    $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+    require 'embulk/command/embulk_main'
+  end
+
+else
+  # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
+  # GEM_HOME is already set by embulk/command/embulk_main.rb
+  ENV.delete('EMBULK_BUNDLE_PATH')
+  user_home = java.lang.System.properties["user.home"] || ENV['HOME']
+  unless user_home
+    raise "HOME environment variable is not set."
+  end
+  ENV['GEM_HOME'] = File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  ENV['GEM_PATH'] = ''
+
+  ENV.delete('BUNDLE_GEMFILE')
+  Gem.clear_paths  # force rubygems to reload GEM_HOME
+
+  $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+  require 'embulk/command/embulk_main'
+end

data/lib/embulk/command/embulk_main.rb

@@ -0,0 +1,2 @@
+require 'embulk/command/embulk_run'
+Embulk.run(ARGV)

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -0,0 +1,170 @@
+module Embulk
+  def self.migrate_plugin(path)
+    migrator = Migrator.new(path)
+
+    if ms = migrator.match("**/build.gradle", /org\.embulk:embulk-core:([\d\.]+)?/)
+      lang = :java
+      from_ver = version(ms[0][1])
+      puts "Detected Java plugin for Embulk #{from_ver}..."
+
+    elsif ms = migrator.match("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/)
+      lang = :ruby
+      from_ver = version(ms[0][1])
+      puts "Detected Ruby plugin for Embulk #{from_ver}..."
+
+    elsif ms = migrator.match("**/*.gemspec", /embulk-/)
+      lang = :ruby
+      from_ver = version("0.1.0")
+      puts "Detected Ruby plugin for unknown Embulk version..."
+
+    else
+      raise "Failed to detect plugin language and dependency version"
+    end
+
+    case lang
+    when :java
+      migrate_java_plugin(migrator, from_ver)
+    when :ruby
+      migrate_ruby_plugin(migrator, from_ver)
+    end
+
+    if migrator.modified_files.empty?
+      puts "Done. No files are modified."
+    else
+      puts "Done. Please check modifieid files."
+    end
+  end
+
+  def self.migrate_java_plugin(migrator, from_ver)
+    if from_ver < version("0.7.0")
+      # rename CommitReport to TaskReport
+      migrator.replace("**/*.java", /(CommitReport)/, "TaskReport")
+      migrator.replace("**/*.java", /(commitReport)/, "taskReport")
+    end
+
+    if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-2\.[12345]\W+/)
+      # gradle < 2.6
+      require 'embulk/data/package_data'
+      data = PackageData.new("new", migrator.path)
+      migrator.write "gradle/wrapper/gradle-wrapper.jar", data.content("java/gradle/wrapper/gradle-wrapper.jar")
+      migrator.write "gradle/wrapper/gradle-wrapper.properties", data.content("java/gradle/wrapper/gradle-wrapper.properties")
+    end
+
+    #
+    # add rules...
+    ##
+
+    # update version at the end
+    migrator.replace("**/build.gradle", /org\.embulk:embulk-(?:core|standards):([\d\.]+)?/, Embulk::VERSION)
+  end
+
+  def self.migrate_ruby_plugin(migrator, from_ver)
+    #
+    # add rules...
+    ##
+
+    migrator.write(".ruby-version", "jruby-9.0.0.0")
+
+    # update version at the end
+    if from_ver <= version("0.1.0")
+      # add add_development_dependency
+      migrator.insert_line("**/*.gemspec", /([ \t]*\w+)\.add_development_dependency/) {|m|
+        "#{m[1]}.add_development_dependency 'embulk', ['~> #{Embulk::VERSION}']"
+      }
+    else
+      migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/, Embulk::VERSION)
+    end
+  end
+
+  private
+
+  def self.version(str)
+    Gem::Version.new(str)
+  end
+
+  class Migrator
+    def initialize(path)
+      @path = path
+      @modified_files = {}
+    end
+
+    attr_reader :path
+
+    def modified_files
+      @modified_files.keys
+    end
+
+    def match(glob, pattern)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        File.read(file).match(pattern)
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def replace(glob, pattern, text=nil)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        data = File.read(file)
+        first = nil
+        pos = 0
+        while pos < data.length
+          m = data.match(pattern, pos)
+          break unless m
+          first ||= m
+          replace = text || yield(m)
+          data = m.pre_match + data[m.begin(0)..(m.begin(1)-1)] + replace + data[m.end(1)..(m.end(0)-1)] + m.post_match
+          pos = m.begin(1) + replace.length + (m.end(0) - m.end(1))
+        end
+        if first
+          modify(file, data)
+        end
+        first
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def insert_line(glob, pattern, text=nil)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        data = File.read(file)
+        if m = data.match(pattern)
+          ln = m.pre_match.split("\n").count
+          replace = text || yield(m)
+          lines = data.split("\n")
+          lines.insert(ln + 1, replace)
+          data = lines.join("\n")
+          modify(file, data)
+          m
+        end
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def data(file)
+      File.read(File.join(@path, file))
+    end
+
+    def write(file, data)
+      modify(File.join(@path, file), data)
+    end
+
+    private
+
+    def modify(path, data)
+      orig = File.read(path) rescue nil
+      if orig != data
+        File.write(path, data)
+        unless @modified_files.has_key?(path)
+          if orig
+            puts "  Modified #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
+          else
+            puts "  Created #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
+          end
+          @modified_files[path] = true
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/embulk/command/embulk_new_plugin.rb

@@ -77,6 +77,7 @@ module Embulk
       when :ruby
         pkg.cp("ruby/Rakefile", "Rakefile")
         pkg.cp("ruby/Gemfile", "Gemfile")
+        pkg.cp("ruby/.ruby-version", ".ruby-version")
         pkg.cp_erb("ruby/gemspec.erb", "#{full_project_name}.gemspec")
         pkg.cp_erb("ruby/#{category}.rb.erb", plugin_path)
 
@@ -104,7 +105,9 @@ module Embulk
         puts "Next steps:"
         puts ""
         puts "  $ cd #{full_project_name}"
-        puts "  $ rake"
+        puts "  $ bundle install                      # install one using rbenv & rbenv-build"
+        puts "  $ bundle exec rake                    # build gem to be released"
+        puts "  $ bundle exec embulk run config.yml   # you can run plugin using this command"
       when :java
         puts "Next steps:"
         puts ""

data/lib/embulk/command/embulk_run.rb

@@ -1,37 +1,26 @@
+require 'embulk'
+
 module Embulk
   def self.run(argv)
-    # default_bundle_path
-    default_bundle_path = nil
-    gemfile_path = ENV['BUNDLE_GEMFILE'].to_s
-    gemfile_path = nil if gemfile_path.empty?
-    default_bundle_path = File.dirname(gemfile_path) if gemfile_path
-
-    # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
-    # GEM_HOME is already set by embulk/command/embulk.rb
-    gem_home = ENV['GEM_HOME'].to_s
-    if gem_home.empty?
-      ENV['GEM_HOME'] = default_gem_home
-      Gem.clear_paths  # force rubygems to reload GEM_HOME
-    end
+    # reset context class loader set by org.jruby.Main.main to nil. embulk manages
+    # multiple classloaders. default classloader should be Plugin.class.getClassloader().
+    java.lang.Thread.current_thread.set_context_class_loader(nil)
 
     # Gem.path is called when GemRunner installs a gem with native extension.
     # Running extconf.rb fails without this hack.
     fix_gem_ruby_path
 
-    # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
-    $LOAD_PATH << Embulk.home('lib')
-
     require 'embulk/version'
 
     i = argv.find_index {|arg| arg !~ /^\-/ }
     unless i
       if argv.include?('--version')
         puts "embulk #{Embulk::VERSION}"
-        exit 0
+        system_exit_success
       end
       usage nil
     end
-    subcmd = argv.slice!(i)
+    subcmd = argv.slice!(i).to_sym
 
     require 'java'
     require 'optparse'
@@ -46,32 +35,25 @@ module Embulk
     classpath_separator = java.io.File.pathSeparator
 
     options = {
-      # use the global ruby runtime (the jruby Runtime running this embulk_run.rb script) for all
-      # ScriptingContainer injected by the org.embulk.command.Runner.
-      useGlobalRubyRuntime: true,
-      systemProperty: {},
+      system_config: {}
     }
 
-    op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
-      # only for help message. implemented at lib/embulk/command/embulk.rb
-    end
-
-    case subcmd.to_sym
-    when :bundle
-      op.remove  # remove --bundle
-      if default_bundle_path
-        op.banner = "Usage: bundle [directory=#{default_bundle_path}]"
-        args = 0..1
-      else
-        op.banner = "Usage: bundle <directory>"
-        args = 1..1
-      end
-
-    when :run
-      op.banner = "Usage: run <config.yml>"
+    java_embed_ops = lambda do
+      op.separator ""
+      op.separator "  Other options:"
       op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
-        options[:logLevel] = level
+        options[:system_config][:log_level] = level
+      end
+      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
+        k, v = kv.split('=', 2)
+        v ||= "true"
+        options[:system_config][k] = v
       end
+    end
+
+    plugin_load_ops = lambda do
+      op.separator ""
+      op.separator "  Plugin load options:"
       op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
         plugin_paths << plugin_path
       end
@@ -81,96 +63,83 @@ module Embulk
       op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
         classpaths.concat classpath.split(classpath_separator)
       end
-      op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
-        options[:nextConfigOutputPath] = path
+      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory (create one using "embulk bundle new" command)') do |path|
+        # only for help message. implemented at lib/embulk/command/embulk_bundle.rb
       end
+    end
+
+    case subcmd
+    when :run
+      op.banner = "Usage: run <config.yml>"
+      op.separator "  Options:"
       op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
-        options[:resumeStatePath] = path
+        options[:resume_state_path] = path
       end
-      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
-        k, v = kv.split('=', 2)
-        v ||= "true"
-        options[:systemProperty][k] = v
+      op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
+        options[:next_config_output_path] = path
       end
+      plugin_load_ops.call
+      java_embed_ops.call
       args = 1..1
 
     when :cleanup
       op.banner = "Usage: cleanup <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
-        options[:logLevel] = level
-      end
-      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
-        plugin_paths << plugin_path
-      end
-      op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
-        load_paths << load_path
-      end
-      op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
-        classpaths.concat classpath.split(classpath_separator)
-      end
+      op.separator "  Options:"
       op.on('-r', '--resume-state PATH', 'Path to a file to write or read resume state') do |path|
-        options[:resumeStatePath] = path
-      end
-      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
-        k, v = kv.split('=', 2)
-        v ||= "true"
-        options[:systemProperty][k] = v
+        options[:resume_state_path] = path
       end
+      plugin_load_ops.call
+      java_embed_ops.call
       args = 1..1
 
     when :preview
       op.banner = "Usage: preview <config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
-        options[:logLevel] = level
-      end
-      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
-        plugin_paths << plugin_path
-      end
-      op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
-        load_paths << load_path
-      end
-      op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
-        classpaths.concat classpath.split(classpath_separator)
-      end
+      op.separator "  Options:"
       op.on('-G', '--vertical', "Use vertical output format", TrueClass) do |b|
-        options[:previewOutputFormat] = "vertical"
-      end
-      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
-        k, v = kv.split('=', 2)
-        v ||= "true"
-        options[:systemProperty][k] = v
+        options[:format] = "vertical"
       end
+      plugin_load_ops.call
+      java_embed_ops.call
       args = 1..1
 
     when :guess
       op.banner = "Usage: guess <partial-config.yml>"
-      op.on('-l', '--log-level LEVEL', 'Log level (error, warn, info, debug or trace)') do |level|
-        options[:logLevel] = level
-      end
+      op.separator "  Options:"
       op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
-        options[:nextConfigOutputPath] = path
-      end
-      op.on('-L', '--load PATH', 'Add a local plugin path') do |plugin_path|
-        plugin_paths << plugin_path
-      end
-      op.on('-I', '--load-path PATH', 'Add ruby script directory path ($LOAD_PATH)') do |load_path|
-        load_paths << load_path
-      end
-      op.on('-C', '--classpath PATH', "Add java classpath separated by #{classpath_separator} (CLASSPATH)") do |classpath|
-        classpaths.concat classpath.split(classpath_separator)
+        options[:next_config_output_path] = path
       end
       op.on('-g', '--guess NAMES', "Comma-separated list of guess plugin names") do |names|
-        (options[:guessPlugins] ||= []).concat names.split(",")
-      end
-      op.on('-X KEY=VALUE', 'Add a performance system config') do |kv|
-        k, v = kv.split('=', 2)
-        v ||= "true"
-        options[:systemProperty][k] = v
+        (options[:system_config][:guess_plugins] ||= []).concat names.split(",")  # TODO
       end
+      plugin_load_ops.call
+      java_embed_ops.call
       args = 1..1
 
+    when :bundle
+      if argv[0] == 'new'
+        usage nil if argv.length != 2
+        new_bundle(argv[1])
+      else
+        gemfile_path = ENV['BUNDLE_GEMFILE'].to_s
+
+        if !gemfile_path.empty?
+          bundle_path = File.dirname(gemfile_path)
+        elsif File.exists?('Gemfile')
+          bundle_path = '.'
+        else
+          system_exit "'#{path}' already exists. You already ran 'embulk bundle new'. Please remove it, or run \"cd #{path}\" and \"embulk bundle\" instead"
+        end
+
+        run_bundler(argv)
+      end
+      system_exit_success
+
+    when :gem
+      require 'rubygems/gem_runner'
+      Gem::GemRunner.new.run argv
+      system_exit_success
+
     when :new
-      op.remove  # remove --bundle
       op.banner = "Usage: new <category> <name>" + %[
 categories:
     ruby-input                 Ruby record input plugin    (like "mysql")
@@ -198,20 +167,16 @@ examples:
 ]
       args = 2..2
 
+    when :migrate
+      op.banner = "Usage: migrate <directory>"
+      args = 1..1
+
     when :selfupdate
-      op.remove  # remove --bundle
       op.on('-f', "Skip corruption check", TrueClass) do |b|
-        options[:force] = true
+        system[:force] = true
       end
       args = 0..0
 
-    when :gem
-      #require 'embulk/gems'
-      #Embulk.add_embedded_gem_path
-      require 'rubygems/gem_runner'
-      Gem::GemRunner.new.run argv
-      exit 0
-
     when :example
       args = 0..1
 
@@ -222,10 +187,10 @@ examples:
     when :irb
       require 'irb'
       IRB.start
-      exit 0
+      system_exit_success
 
     else
-      usage "Unknown subcommand #{subcmd.dump}."
+      usage "Unknown subcommand #{subcmd.to_s.dump}."
     end
 
     begin
@@ -237,76 +202,9 @@ examples:
       usage_op op, e.to_s
     end
 
-    case subcmd.to_sym
-    when :bundle
-      path = argv[0] || default_bundle_path
-
-      require 'fileutils'
-      require 'rubygems/gem_runner'
-      setup_plugin_paths(plugin_paths)
-      setup_load_paths(load_paths)
-      setup_classpaths(classpaths)
-
-      unless File.exists?(path)
-        puts "Initializing #{path}..."
-        FileUtils.mkdir_p File.dirname(path)
-        begin
-          success = false
-
-          # copy embulk/data/bundle/ directory
-          require 'embulk/data/package_data'
-          pkg = PackageData.new("bundle", path)
-          %w[.bundle/config embulk/input/example.rb embulk/output/example.rb embulk/filter/example.rb Gemfile].each do |file|
-            pkg.cp(file, file)
-          end
-
-          ## TODO this is disabled for now. enable this if you want to use
-          ## create bin/embulk
-          #bin_embulk_path = File.join(path, 'bin', 'embulk')
-          #FileUtils.mkdir_p File.dirname(bin_embulk_path)
-          #require 'embulk/command/embulk_generate_bin'  # defines Embulk.generate_bin
-          #File.open(bin_embulk_path, 'wb', 0755) {|f| f.write Embulk.generate_bin(bundle_path: :here) }
-
-          # install bundler
-          setup_gem_paths(path)
-          Gem::GemRunner.new.run %w[install bundler]
-
-          success = true
-        rescue Gem::SystemExitException => e
-          raise e if e.exit_code != 0
-          success = true
-        ensure
-          FileUtils.rm_rf path unless success
-        end
-      else
-        setup_gem_paths(path)
-      end
-
-      ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(path, "Gemfile")
-      Dir.chdir(path) do
-        begin
-          require 'bundler'
-        rescue LoadError
-          require 'rubygems/gem_runner'
-          begin
-            puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S.%3N %z")}: installing bundler..."
-            Gem::GemRunner.new.run %w[install bundler]
-          rescue Gem::SystemExitException => e
-            raise e if e.exit_code != 0
-          end
-          Gem.clear_paths
-          require 'bundler'
-        end
-        require 'bundler/friendly_errors'
-        require 'bundler/cli'
-        Bundler.with_friendly_errors do
-          # run > bundle install
-          Bundler::CLI.start(%w[install], debug: true)
-        end
-      end
-
+    case subcmd
     when :example
-      require_relative 'embulk_example'
+      require 'embulk/command/embulk_example'
       path = ARGV[0] || "embulk-example"
       puts "Creating #{path} directory..."
       Embulk.create_example(path)
@@ -348,26 +246,33 @@ examples:
       require 'embulk/command/embulk_new_plugin'
       Embulk.new_plugin(name, language, category)
 
+    when :migrate
+      path = ARGV[0]
+      require 'embulk/command/embulk_migrate_plugin'
+      Embulk.migrate_plugin(path)
+
     when :selfupdate
       require 'embulk/command/embulk_selfupdate'
-      Embulk.selfupdate(options)
+      Embulk.selfupdate(system)
 
     else
       require 'json'
 
-      begin
-        java.lang.Class.forName('org.embulk.command.Runner')
-      rescue java.lang.ClassNotFoundException
-        # load classpath
-        Embulk.require_classpath
-      end
+      Embulk.setup(options.delete(:system_config))
 
       setup_plugin_paths(plugin_paths)
       setup_load_paths(load_paths)
       setup_classpaths(classpaths)
 
       begin
-        org.embulk.command.Runner.new(options.to_json).main(subcmd, argv.to_java(:string))
+        case subcmd
+        when :guess
+          Embulk::Runner.guess(argv[0], options)
+        when :preview
+          Embulk::Runner.preview(argv[0], options)
+        when :run
+          Embulk::Runner.run(argv[0], options)
+        end
       rescue => ex
         print_exception(ex)
         puts ""
@@ -377,25 +282,6 @@ examples:
     end
   end
 
-  def self.home(dir)
-    jar, resource = __FILE__.split("!")
-    if resource && jar =~ /^file:/
-      home = resource.split("/")[0..-3].join("/")
-      "#{jar}!#{home}/#{dir}"
-    else
-      home = File.expand_path('../../..', File.dirname(__FILE__))
-      File.join(home, dir)
-    end
-  end
-
-  def self.require_classpath
-    classpath_dir = Embulk.home("classpath")
-    jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
-    jars.each do |jar|
-      require File.join(classpath_dir, jar)
-    end
-  end
-
   def self.default_gem_home
     if RUBY_PLATFORM =~ /java/i
       user_home = java.lang.System.properties["user.home"]
@@ -426,13 +312,6 @@ examples:
     end
   end
 
-  def self.setup_gem_paths(path)
-    # install bundler gem here & use bundler installed here
-    ENV['GEM_HOME'] = File.expand_path File.join(path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
-    ENV['GEM_PATH'] = ''
-    Gem.clear_paths  # force rubygems to reload GEM_HOME
-  end
-
   def self.setup_plugin_paths(plugin_paths)
     plugin_paths.each do |path|
       unless File.directory?(path)
@@ -444,9 +323,12 @@ examples:
       end
       specs.each do |spec|
         gem_path = File.dirname(spec)
-        stub = Gem::StubSpecification.new(spec)
-        stub.define_singleton_method(:full_gem_path) { gem_path }
-        Gem::Specification.add_spec(stub)
+        Dir.chdir(path) do
+          # cd to path because spec could include `git ...`
+          stub = Gem::StubSpecification.new(spec)
+          stub.define_singleton_method(:full_gem_path) { gem_path }
+          Gem::Specification.add_spec(stub)
+        end
       end
     end
   end
@@ -465,6 +347,79 @@ examples:
     end
   end
 
+  def self.new_bundle(path)
+    require 'fileutils'
+    require 'rubygems/gem_runner'
+
+    if File.exists?(path)
+      error = "'#{path}' already exists. You already ran 'embulk bundle new'. Please remove it, or run \"cd #{path}\" and \"embulk bundle\" instead"
+      STDERR.puts error
+      raise SystemExit.new(1, error)
+    end
+
+    puts "Initializing #{path}..."
+    FileUtils.mkdir_p path
+    begin
+      success = false
+
+      # copy embulk/data/bundle/ contents
+      require 'embulk/data/package_data'
+      pkg = PackageData.new("bundle", path)
+      %w[Gemfile .ruby-version .bundle/config embulk/input/example.rb embulk/output/example.rb embulk/filter/example.rb].each do |file|
+        pkg.cp(file, file)
+      end
+
+      # run the first bundle-install
+      Dir.chdir(path) do
+        run_bundler(['install', '--path', '.'])
+      end
+
+      success = true
+    rescue Gem::SystemExitException => e
+      raise e if e.exit_code != 0
+      success = true
+    ensure
+      FileUtils.rm_rf path unless success
+    end
+  end
+
+  def self.run_bundler(argv)
+    # try to load bundler from LOAD_PATH and ./jruby
+    begin
+      require 'bundler'
+      bundler_provided = true
+    rescue LoadError
+      ENV['GEM_HOME'] = File.expand_path File.join(".", Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+      ENV['GEM_PATH'] = ''
+      ENV.delete('BUNDLE_GEMFILE')
+      Gem.clear_paths  # force rubygems to reload GEM_HOME
+      begin
+        require 'bundler'
+        bundler_provided = true
+      rescue LoadError
+      end
+    end
+
+    unless bundler_provided
+      # install bundler to the path (bundler is not included in embulk-core.jar)
+      require 'rubygems/gem_runner'
+      begin
+        puts "Installing bundler..."
+        Gem::GemRunner.new.run %w[install bundler]
+      rescue Gem::SystemExitException => e
+        raise e if e.exit_code != 0
+      end
+      Gem.clear_paths
+      require 'bundler'
+    end
+
+    require 'bundler/friendly_errors'
+    require 'bundler/cli'
+    Bundler.with_friendly_errors do
+      Bundler::CLI.start(argv, debug: true)
+    end
+  end
+
   def self.usage(message)
     STDERR.puts "Embulk v#{Embulk::VERSION}"
     STDERR.puts "usage: <command> [--options]"
@@ -476,24 +431,21 @@ examples:
     STDERR.puts "   gem       <install | list | help>                  # install a plugin or show installed plugins."
     STDERR.puts "                                                      # plugin path is #{ENV['GEM_HOME']}"
     STDERR.puts "   new       <category> <name>                        # generates new plugin template"
+    STDERR.puts "   migrate   <path>                                   # modify plugin code to use the latest Embulk plugin API"
     STDERR.puts "   example   [path]                                   # creates an example config file and csv file to try embulk."
     STDERR.puts "   selfupdate                                         # upgrades embulk to the latest released version."
     STDERR.puts ""
     if message
-      STDERR.puts "error: #{message}"
+      system_exit "error: #{message}"
     else
-      STDERR.puts "Use \`<command> --help\` to see description of the commands."
+      system_exit "Use \`<command> --help\` to see description of the commands."
     end
-    exit 1
   end
 
   def self.usage_op(op, message)
     STDERR.puts op.help
     STDERR.puts
-    if message
-      STDERR.puts message
-    end
-    exit 1
+    system_exit message
   end
 
   def self.print_exception(ex)
@@ -506,4 +458,13 @@ examples:
       end
     end
   end
+
+  def self.system_exit(message=nil)
+    STDERR.puts message if message
+    raise SystemExit.new(1, message)
+  end
+
+  def self.system_exit_success
+    raise SystemExit.new(0)
+  end
 end