embulk 0.1.0

data/lib/embulk/command/embulk.rb

@@ -0,0 +1,39 @@
+bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
+bundle_path = nil if bundle_path.empty?
+
+# search -b, --bundle BUNDLE_DIR
+bundle_path_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+if bundle_path_index
+  bundle_path = ARGV.slice!(bundle_path_index, 2)[1]
+end
+
+if bundle_path
+  # use bundler installed at bundle_path
+  ENV['GEM_HOME'] = File.expand_path File.join(bundle_path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  ENV['GEM_PATH'] = ''
+  Gem.clear_paths  # force rubygems to reload GEM_HOME
+
+  ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
+  require 'bundler'
+  Bundler.load.setup_environment
+  require 'bundler/setup'
+  # since here, `require` may load files of different (newer) embulk versions
+  # especially following 'embulk/command/embulk_run'.
+
+  $LOAD_PATH << File.expand_path(bundle_path)  # for local plugins
+
+  begin
+    require 'embulk/command/embulk_run'
+  rescue LoadError
+    require_relative 'embulk_run'
+  end
+
+else
+  # GEM_HOME is set by embulk_run.rb
+  ENV.delete('GEM_HOME')
+  ENV.delete('GEM_PATH')
+  ENV.delete('BUNDLE_GEMFILE')
+  require_relative 'embulk_run'
+end
+
+Embulk.run(ARGV)

data/lib/embulk/command/embulk_example.rb

@@ -0,0 +1,32 @@
+module Embulk
+  def self.create_example(path)
+    require 'fileutils'
+    require 'zlib'
+
+    puts "  Creating #{path}/"
+    FileUtils.mkdir_p File.join(path, 'csv')
+    puts "  Creating #{path}/csv/"
+
+    puts "  Creating #{path}/csv/sample_01.csv.gz"
+    Zlib::GzipWriter.open(File.join(path, 'csv', 'sample_01.csv.gz')) do |f|
+      f.write <<EOF
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,embulk core
+4,11270,2015-01-29 11:54:36,20150129,"Embulk ""csv"" parser plugin"
+EOF
+    end
+
+    puts "  Creating #{path}/example.yml"
+    File.open(File.join(path, 'example.yml'), 'w') do |f|
+      f.write <<EOF
+in:
+  type: file
+  paths: [#{File.expand_path File.join(path, 'csv')}]
+out:
+  type: stdout
+EOF
+    end
+  end
+end

data/lib/embulk/command/embulk_generate_bin.rb

@@ -0,0 +1,62 @@
+module Embulk
+  def self.generate_bin(options={})
+    jruby_jar_path = org.jruby.Main.java_class.protection_domain.code_source.location.to_s
+    if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
+      resource_class = org.embulk.command.Runner.java_class
+      ruby_script_path = resource_class.resource("/embulk/command/embulk.rb").to_s
+    else
+      ruby_script_path = File.join(File.dirname(__FILE__), 'embulk.rb')
+    end
+    java_home = java.lang.System.properties['java.home']
+
+    generate_bin_data(jruby_jar_path, ruby_script_path, options.merge(java_home: java_home))
+  end
+
+  def self.generate_bin_data(jruby_jar_path, ruby_script_path, options={})
+    if java_home = options[:java_home]
+      java_home_script = %{export JAVA_HOME='#{java_home}'}
+      java_path = %{"$JAVA_HOME"/bin/java}
+    else
+      java_home_script = %{}
+      java_path = %{java}
+    end
+
+    # TODO parse -D options to set them to java
+    shell_script = <<EOF
+#!/bin/sh
+=begin 2>/dev/null
+#{java_home_script}
+exec #{java_path} -classpath "$0" org.jruby.Main "$0" "$@"
+exit 127
+=end
+EOF
+
+    if options[:bundle_path] == :here
+      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = File.expand_path('..', File.dirname(__FILE__))}
+    elsif path = options[:bundle_path]
+      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = '#{path}'}
+    else
+      bundle_path_script = b ''
+    end
+
+    ruby_init_script = b <<EOF
+#{bundle_path_script}
+ENV.delete 'GEM_HOME'
+ENV.delete 'GEM_PATH'
+EOF
+
+    ruby_script = b(File.read(ruby_script_path))
+    if i = ruby_script.index(b("\n__END__\n"))
+      # delete contents after __END__
+      ruby_script = ruby_script[0, i]
+    end
+
+    jruby_jar = b(File.read(jruby_jar_path))
+
+    data = shell_script << ruby_init_script + ruby_script << b("\n__END__\n") << jruby_jar
+  end
+
+  def self.b(s)
+    s.force_encoding('ASCII-8BIT')
+  end
+end

data/lib/embulk/command/embulk_run.rb

@@ -0,0 +1,243 @@
+module Embulk
+  def self.run(argv)
+    # default_bundle_path
+    default_bundle_path = nil
+    gemfile_path = ENV['BUNDLE_GEMFILE'].to_s
+    gemfile_path = nil if gemfile_path.empty?
+    default_bundle_path = File.dirname(gemfile_path) if gemfile_path
+
+    # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
+    # GEM_HOME is already set by embulk/command/embulk.rb
+    gem_home = ENV['GEM_HOME'].to_s
+    if gem_home.empty?
+      ENV['GEM_HOME'] = File.expand_path File.join(ENV['HOME'], '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+      Gem.clear_paths  # force rubygems to reload GEM_HOME
+    end
+
+    i = argv.find_index {|arg| arg !~ /^\-/ }
+    usage nil unless i
+    subcmd = argv.slice!(i)
+
+    load_paths = []
+    options = {}
+
+    # to make sure org.embulk.jruby.JRubyScriptingModule can require 'embulk/java/bootstrap'
+    $LOAD_PATH << Embulk.home('lib')
+
+    require 'optparse'
+    op = OptionParser.new
+
+    op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
+      # only for help message. implemented at lib/embulk/command/embulk.rb
+    end
+
+    case subcmd.to_sym
+    when :bundle
+      if default_bundle_path
+        op.banner = "Usage: bundle [directory=#{default_bundle_path}]"
+        args = 0..1
+      else
+        op.banner = "Usage: bundle <directory>"
+        args = 1..1
+      end
+
+    when :run
+      op.banner = "Usage: run <config.yml>"
+      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
+      end
+      op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
+        load_paths << load_path
+      end
+      op.on('-o', '--output PATH', 'Path to a file to write the next configuration') do |path|
+        options[:nextConfigOutputPath] = path
+      end
+      args = 1..1
+
+    when :preview
+      op.banner = "Usage: preview <config.yml>"
+      op.on('-b', '--bundle BUNDLE_DIR', 'Path to a Gemfile directory') do |path|
+      end
+      op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
+        load_paths << load_path
+      end
+      args = 1..1
+
+    when :guess
+      op.banner = "Usage: guess <partial-config.yml>"
+      op.on('-o', '--output PATH', 'Path to a file to write the guessed configuration') do |path|
+        options[:nextConfigOutputPath] = path
+      end
+      op.on('-I', '--load-path PATH', 'Add ruby script directory path or jar file path') do |load_path|
+        load_paths << load_path
+      end
+      args = 1..1
+
+    #when :generate  # or :new
+      # TODO create plugin templates
+
+    when :gem
+      require 'rubygems/gem_runner'
+      Gem::GemRunner.new.run argv
+      exit 0
+
+    when :example
+      args = 0..1
+
+    when :exec
+      exec *argv
+      exit 127
+
+    else
+      usage "Unknown subcommand #{subcmd.dump}."
+    end
+
+    begin
+      op.parse!(argv)
+      unless args.include?(argv.length)
+        usage nil
+      end
+    rescue => e
+      usage e.to_s
+    end
+
+    case subcmd.to_sym
+    when :bundle
+      path = argv[0] || default_bundle_path
+
+      require 'fileutils'
+      require 'rubygems/gem_runner'
+      setup_load_paths(load_paths)
+
+      unless File.exists?(path)
+        puts "Initializing #{path}..."
+        FileUtils.mkdir_p File.dirname(path)
+        begin
+          success = false
+
+          # copy embulk/data/bundle/ directory
+          if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
+            # data is in embulk-core jar
+            resource_class = org.embulk.command.Runner.java_class
+            %w[.bundle/config embulk/input_example.rb embulk/output_example.rb Gemfile Gemfile.lock].each do |file|  # TODO get file list from the jar
+              url = resource_class.resource("/embulk/data/bundle/#{file}").to_s
+              dst = File.join(path, file)
+              FileUtils.mkdir_p File.dirname(dst)
+              FileUtils.cp(url, dst)
+            end
+          else
+            #tmpl = File.join(File.dirname(__FILE__), '../data/bundle')
+            tmpl = File.join(Embulk.home('lib'), 'embulk', 'data', 'bundle')
+            FileUtils.cp_r tmpl, path
+          end
+
+          ## TODO this is disabled for now. enable this if you want to use
+          ## create bin/embulk
+          #bin_embulk_path = File.join(path, 'bin', 'embulk')
+          #FileUtils.mkdir_p File.dirname(bin_embulk_path)
+          #require 'embulk/command/embulk_generate_bin'  # defines Embulk.generate_bin
+          #File.open(bin_embulk_path, 'wb', 0755) {|f| f.write Embulk.generate_bin(bundle_path: :here) }
+
+          # install bundler
+          setup_gem_paths(path)
+          Gem::GemRunner.new.run %w[install bundler]
+
+          success = true
+        rescue Gem::SystemExitException => e
+          raise e if e.exit_code != 0
+          success = true
+        ensure
+          FileUtils.rm_rf path unless success
+        end
+      else
+        setup_gem_paths(path)
+      end
+
+      ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(path, "Gemfile")
+      Dir.chdir(path) do
+        require 'bundler'
+        require 'bundler/friendly_errors'
+        require 'bundler/cli'
+        Bundler.with_friendly_errors do
+          # run > bundle install
+          Bundler::CLI.start(%w[install], debug: true)
+        end
+      end
+
+    when :example
+      require_relative 'embulk_example'
+      path = ARGV[0] || "embulk-example"
+      puts "Creating #{path} directory..."
+      Embulk.create_example(path)
+      puts ""
+      puts "Run following subcommands to try embulk:"
+      puts ""
+      puts "   1. guess #{File.join(path, 'example.yml')} -o config.yml"
+      puts "   2. preview config.yml"
+      puts "   3. run config.yml"
+      puts ""
+
+    else
+      require 'json'
+
+      begin
+        java.lang.Class.forName('org.embulk.command.Runner')
+      rescue java.lang.ClassNotFoundException
+        # load classpath
+        classpath_dir = Embulk.home('classpath')
+        jars = Dir.entries(classpath_dir).select {|f| f =~ /\.jar$/ }.sort
+        jars.each do |jar|
+          require File.join(classpath_dir, jar)
+        end
+      end
+
+      setup_load_paths(load_paths)
+
+      org.embulk.command.Runner.new(options.to_json).main(subcmd, argv.to_java(:string))
+    end
+  end
+
+  def self.home(dir)
+    home = File.expand_path('../../..', File.dirname(__FILE__))
+    File.join(home, dir)
+  end
+
+  private
+
+  def self.setup_gem_paths(path)
+    # install bundler gem here & use bundler installed here
+    ENV['GEM_HOME'] = File.expand_path File.join(path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+    ENV['GEM_PATH'] = ''
+    Gem.clear_paths  # force rubygems to reload GEM_HOME
+  end
+
+  def self.setup_load_paths(load_paths)
+    load_paths.each do |load_path|
+      if File.file?(load_path)
+        # jar files
+        require File.expand_path(load_path)
+      else
+        # ruby script directory (add at the beginning of $LOAD_PATH to make it highest priority)
+        $LOAD_PATH.unshift File.expand_path(load_path)
+      end
+    end
+  end
+
+  def self.usage(message)
+    STDERR.puts "usage: <command> [--options]"
+    STDERR.puts "commands:"
+    STDERR.puts "   bundle    [directory]                              # create or update plugin environment."
+    STDERR.puts "   run       <config.yml>                             # run a bulk load transaction."
+    STDERR.puts "   preview   <config.yml>                             # dry-run the bulk load without output and show preview."
+    STDERR.puts "   guess     <partial-config.yml> -o <output.yml>     # guess missing parameters to create a complete configuration file."
+    STDERR.puts "   gem       <install | list | help>                  # install a plugin or show installed plugins."
+    STDERR.puts "                                                      # plugin path is #{ENV['GEM_HOME']}"
+    STDERR.puts "   example   [path]                                   # creates an example config file and csv file to try embulk."
+    STDERR.puts ""
+    if message
+      STDERR.puts "error: #{message}"
+    else
+      STDERR.puts "Use \`<command> --help\` to see description of the commands."
+    end
+    exit 1
+  end
+end

data/lib/embulk/data/bundle/.bundle/config

@@ -0,0 +1,3 @@
+---
+BUNDLE_PATH: .
+BUNDLE_DISABLE_SHARED_GEMS: '1'

data/lib/embulk/data/bundle/Gemfile

@@ -0,0 +1,31 @@
+source 'https://rubygems.org/'
+
+# Plugin bundle:
+#
+# a) Using plugins released on RubyGems:
+#   You can write plugin names (and versions) to this file to include
+#   the plugins released on RubyGems. To use those plugins,
+#
+#   1. modify this Gemfile. Examples are at the end of this file.
+#   2. update Gemfile.lock using: embulk bundle <this dir>
+#   3. run embulk with -b option: embulk run -b <this dir> example.yml
+#
+# b) Using simple single-file ruby plugins:
+#   You can put a simple ruby plugins to <this dir>/embulk directory.
+#   When you run embulk with -b <this dir> option, embulk loads them:
+#
+#   $ embulk run -b <this dir> example.yml
+#
+#   See <this dir>/embulk/* files for example. When the plugin works,
+#   you can release the plugin to RubyGems with "embulk-plugin-foobar"
+#   name to deploy them to servers.
+#
+
+# Plugins
+#
+#gem 'embulk-plugin-foo'
+#gem 'embulk-plugin-bar', '~> 0.1.0'
+#gem 'embulk-plugin-baz', '= 0.2.0'
+#gem 'embulk-plugin-xyz', '>= 0.3.0'
+#
+

data/lib/embulk/data/bundle/Gemfile.lock

@@ -0,0 +1,8 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+
+PLATFORMS
+  java
+
+DEPENDENCIES

data/lib/embulk/data/bundle/embulk/input_example.rb

@@ -0,0 +1,40 @@
+module Embulk
+
+  class InputExample < InputPlugin
+    # input plugin file name must be: embulk/output_<name>.rb
+    Plugin.register_input('example', self)
+
+    def self.transaction(config, &control)
+      task = {
+        'message' => config.param('message', :string, default: nil)
+      }
+      threads = config.param('threads', :integer, default: 2)
+
+      columns = [
+        Column.new(0, 'col0', :long),
+        Column.new(1, 'col1', :double),
+        Column.new(2, 'col2', :string),
+      ]
+
+      puts "Example input started."
+      commit_reports = yield(task, columns, threads)
+      puts "Example input finished. Commit reports = #{commit_reports.to_json}"
+
+      return {}
+    end
+
+    def self.run(task, schema, index, page_builder)
+      puts "Example input thread #{index}..."
+
+      10.times do |i|
+        page_builder.add([i, 10.0, "example"])
+      end
+      page_builder.finish  # don't forget to call finish :-)
+
+      commit_report = {
+      }
+      return commit_report
+    end
+  end
+
+end