embulk 0.7.0-java

data/lib/embulk/command/embulk_bundle.rb

@@ -0,0 +1,56 @@
+
+bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
+bundle_path = nil if bundle_path.empty?
+
+# search -b, --bundle BUNDLE_DIR
+bundle_path_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+if bundle_path_index
+  bundle_path = ARGV.slice!(bundle_path_index, 2)[1]
+end
+
+if bundle_path
+  # use bundler installed at bundle_path
+  ENV['EMBULK_BUNDLE_PATH'] = bundle_path
+  ENV['GEM_HOME'] = File.expand_path File.join(bundle_path, Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  ENV['GEM_PATH'] = ''
+  Gem.clear_paths  # force rubygems to reload GEM_HOME
+
+  ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
+
+  begin
+    require 'bundler'
+  rescue LoadError => e
+    raise "#{e}\nBundler is not installed. Did you run \`$ embulk bundle #{bundle_path}\` ?"
+  end
+  Bundler.load.setup_environment
+  require 'bundler/setup'
+  # since here, `require` may load files of different (newer) embulk versions
+  # especially following 'embulk/command/embulk_main'.
+
+  # add bundle directory path to load local plugins at ./embulk
+  $LOAD_PATH << File.expand_path(bundle_path)
+
+  begin
+    require 'embulk/command/embulk_main'
+  rescue LoadError
+    $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+    require 'embulk/command/embulk_main'
+  end
+
+else
+  # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
+  # GEM_HOME is already set by embulk/command/embulk_main.rb
+  ENV.delete('EMBULK_BUNDLE_PATH')
+  user_home = java.lang.System.properties["user.home"] || ENV['HOME']
+  unless user_home
+    raise "HOME environment variable is not set."
+  end
+  ENV['GEM_HOME'] = File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+  ENV['GEM_PATH'] = ''
+
+  ENV.delete('BUNDLE_GEMFILE')
+  Gem.clear_paths  # force rubygems to reload GEM_HOME
+
+  $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+  require 'embulk/command/embulk_main'
+end

data/lib/embulk/command/embulk_example.rb

@@ -0,0 +1,32 @@
+module Embulk
+  def self.create_example(path)
+    require 'fileutils'
+    require 'zlib'
+
+    puts "  Creating #{path}/"
+    FileUtils.mkdir_p File.join(path, 'csv')
+    puts "  Creating #{path}/csv/"
+
+    puts "  Creating #{path}/csv/sample_01.csv.gz"
+    Zlib::GzipWriter.open(File.join(path, 'csv', 'sample_01.csv.gz')) do |f|
+      f.write <<EOF
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL
+EOF
+    end
+
+    puts "  Creating #{path}/example.yml"
+    File.open(File.join(path, 'example.yml'), 'w') do |f|
+      f.write <<EOF
+in:
+  type: file
+  path_prefix: "#{File.expand_path File.join(path, 'csv', 'sample_')}"
+out:
+  type: stdout
+EOF
+    end
+  end
+end

data/lib/embulk/command/embulk_generate_bin.rb

@@ -0,0 +1,62 @@
+module Embulk
+  def self.generate_bin(options={})
+    jruby_jar_path = org.jruby.Main.java_class.protection_domain.code_source.location.to_s
+    if __FILE__ =~ /^classpath:/ || __FILE__.include?('!/')
+      resource_class = org.embulk.command.Runner.java_class
+      ruby_script_path = resource_class.resource("/embulk/command/embulk.rb").to_s
+    else
+      ruby_script_path = File.join(File.dirname(__FILE__), 'embulk.rb')
+    end
+    java_home = java.lang.System.properties['java.home']
+
+    generate_bin_data(jruby_jar_path, ruby_script_path, options.merge(java_home: java_home))
+  end
+
+  def self.generate_bin_data(jruby_jar_path, ruby_script_path, options={})
+    if java_home = options[:java_home]
+      java_home_script = %{export JAVA_HOME='#{java_home}'}
+      java_path = %{"$JAVA_HOME"/bin/java}
+    else
+      java_home_script = %{}
+      java_path = %{java}
+    end
+
+    # TODO parse -D options to set them to java
+    shell_script = <<EOF
+#!/bin/sh
+=begin 2>/dev/null
+#{java_home_script}
+exec #{java_path} -classpath "$0" org.jruby.Main "$0" "$@"
+exit 127
+=end
+EOF
+
+    if options[:bundle_path] == :here
+      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = File.expand_path('..', File.dirname(__FILE__))}
+    elsif path = options[:bundle_path]
+      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = '#{path}'}
+    else
+      bundle_path_script = b ''
+    end
+
+    ruby_init_script = b <<EOF
+#{bundle_path_script}
+ENV.delete 'GEM_HOME'
+ENV.delete 'GEM_PATH'
+EOF
+
+    ruby_script = b(File.read(ruby_script_path))
+    if i = ruby_script.index(b("\n__END__\n"))
+      # delete contents after __END__
+      ruby_script = ruby_script[0, i]
+    end
+
+    jruby_jar = b(File.read(jruby_jar_path))
+
+    data = shell_script << ruby_init_script + ruby_script << b("\n__END__\n") << jruby_jar
+  end
+
+  def self.b(s)
+    s.force_encoding('ASCII-8BIT')
+  end
+end

data/lib/embulk/command/embulk_main.rb

@@ -0,0 +1,2 @@
+require 'embulk/command/embulk_run'
+Embulk.run(ARGV)

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -0,0 +1,170 @@
+module Embulk
+  def self.migrate_plugin(path)
+    migrator = Migrator.new(path)
+
+    if ms = migrator.match("**/build.gradle", /org\.embulk:embulk-core:([\d\.]+)?/)
+      lang = :java
+      from_ver = version(ms[0][1])
+      puts "Detected Java plugin for Embulk #{from_ver}..."
+
+    elsif ms = migrator.match("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/)
+      lang = :ruby
+      from_ver = version(ms[0][1])
+      puts "Detected Ruby plugin for Embulk #{from_ver}..."
+
+    elsif ms = migrator.match("**/*.gemspec", /embulk-/)
+      lang = :ruby
+      from_ver = version("0.1.0")
+      puts "Detected Ruby plugin for unknown Embulk version..."
+
+    else
+      raise "Failed to detect plugin language and dependency version"
+    end
+
+    case lang
+    when :java
+      migrate_java_plugin(migrator, from_ver)
+    when :ruby
+      migrate_ruby_plugin(migrator, from_ver)
+    end
+
+    if migrator.modified_files.empty?
+      puts "Done. No files are modified."
+    else
+      puts "Done. Please check modifieid files."
+    end
+  end
+
+  def self.migrate_java_plugin(migrator, from_ver)
+    if from_ver < version("0.7.0")
+      # rename CommitReport to TaskReport
+      migrator.replace("**/*.java", /(CommitReport)/, "TaskReport")
+      migrator.replace("**/*.java", /(commitReport)/, "taskReport")
+    end
+
+    if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-2\.[12345]\W+/)
+      # gradle < 2.6
+      require 'embulk/data/package_data'
+      data = PackageData.new("new", migrator.path)
+      migrator.write "gradle/wrapper/gradle-wrapper.jar", data.content("java/gradle/wrapper/gradle-wrapper.jar")
+      migrator.write "gradle/wrapper/gradle-wrapper.properties", data.content("java/gradle/wrapper/gradle-wrapper.properties")
+    end
+
+    #
+    # add rules...
+    ##
+
+    # update version at the end
+    migrator.replace("**/build.gradle", /org\.embulk:embulk-(?:core|standards):([\d\.]+)?/, Embulk::VERSION)
+  end
+
+  def self.migrate_ruby_plugin(migrator, from_ver)
+    #
+    # add rules...
+    ##
+
+    migrator.write(".ruby-version", "jruby-9.0.0.0")
+
+    # update version at the end
+    if from_ver <= version("0.1.0")
+      # add add_development_dependency
+      migrator.insert_line("**/*.gemspec", /([ \t]*\w+)\.add_development_dependency/) {|m|
+        "#{m[1]}.add_development_dependency 'embulk', ['~> #{Embulk::VERSION}']"
+      }
+    else
+      migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/, Embulk::VERSION)
+    end
+  end
+
+  private
+
+  def self.version(str)
+    Gem::Version.new(str)
+  end
+
+  class Migrator
+    def initialize(path)
+      @path = path
+      @modified_files = {}
+    end
+
+    attr_reader :path
+
+    def modified_files
+      @modified_files.keys
+    end
+
+    def match(glob, pattern)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        File.read(file).match(pattern)
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def replace(glob, pattern, text=nil)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        data = File.read(file)
+        first = nil
+        pos = 0
+        while pos < data.length
+          m = data.match(pattern, pos)
+          break unless m
+          first ||= m
+          replace = text || yield(m)
+          data = m.pre_match + data[m.begin(0)..(m.begin(1)-1)] + replace + data[m.end(1)..(m.end(0)-1)] + m.post_match
+          pos = m.begin(1) + replace.length + (m.end(0) - m.end(1))
+        end
+        if first
+          modify(file, data)
+        end
+        first
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def insert_line(glob, pattern, text=nil)
+      ms = Dir[File.join(@path, glob)].map do |file|
+        data = File.read(file)
+        if m = data.match(pattern)
+          ln = m.pre_match.split("\n").count
+          replace = text || yield(m)
+          lines = data.split("\n")
+          lines.insert(ln + 1, replace)
+          data = lines.join("\n")
+          modify(file, data)
+          m
+        end
+      end.compact
+      return nil if ms.empty?
+      ms
+    end
+
+    def data(file)
+      File.read(File.join(@path, file))
+    end
+
+    def write(file, data)
+      modify(File.join(@path, file), data)
+    end
+
+    private
+
+    def modify(path, data)
+      orig = File.read(path) rescue nil
+      if orig != data
+        File.write(path, data)
+        unless @modified_files.has_key?(path)
+          if orig
+            puts "  Modified #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
+          else
+            puts "  Created #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
+          end
+          @modified_files[path] = true
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/embulk/command/embulk_new_plugin.rb

@@ -0,0 +1,124 @@
+module Embulk
+  def self.new_plugin(name, language, category)
+    require 'embulk/data/package_data'
+    require 'embulk/version'
+    require 'fileutils'
+
+    embulk_category = category
+    embulk_category = :input if category == :file_input
+    embulk_category = :output if category == :file_output
+
+    name = name.gsub(/[^a-zA-Z0-9_]+/, '_')  # replace '-' to '_'
+
+    full_project_name = "embulk-#{embulk_category}-#{name}"
+    plugin_dir = "lib/embulk"
+    plugin_path = "#{plugin_dir}/#{embulk_category}/#{name}.rb"
+
+    if File.exist?(full_project_name)
+      raise "./#{full_project_name} already exists. Please delete it first."
+    end
+    FileUtils.mkdir_p(full_project_name)
+
+    puts "Creating #{full_project_name}/"
+
+    success = false
+    begin
+      #
+      # Generate gemspec
+      #
+      author = `git config user.name`.strip
+      author = "YOUR_NAME" if author.empty?
+      email = `git config user.email`.strip
+      email = "YOUR_NAME" if email.empty?
+
+      # variables used in erb templates
+      ruby_class_name = name.split('_').map {|a| a.capitalize }.join
+      java_iface_name = category.to_s.split('_').map {|a| a.capitalize }.join
+      java_class_name = name.split('_').map {|a| a.capitalize }.join + java_iface_name + "Plugin"
+      java_package_name = "org.embulk.#{embulk_category}.#{name}"
+      display_name = name.split('_').map {|a| a.capitalize }.join(' ')
+      display_category = category.to_s.gsub('_', ' ')
+
+      extra_guess_erb = {}
+
+      case category
+      when :input
+        description = %[Loads records from #{display_name}.]
+      when :file_input
+        description = %[Reads files stored on #{display_name}.]
+      when :parser
+        description = %[Parses #{display_name} files read by other file input plugins.]
+        extra_guess_erb["ruby/parser_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
+      when :decoder
+        description = %[Decodes #{display_name}-encoded files read by other file input plugins.]
+        extra_guess_erb["ruby/decoder_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
+      when :output
+        description = %[Dumps records to #{display_name}.]
+      when :file_output
+        description = %[Stores files on #{display_name}.]
+      when :formatter
+        description = %[Formats #{display_name} files for other file output plugins.]
+      when :encoder
+        description = %[Encodes files using #{display_name} for other file output plugins.]
+      when :filter
+        description = %[#{display_name}]
+      end
+
+      #
+      # Generate project repository
+      #
+      pkg = Embulk::PackageData.new("new", full_project_name, binding())
+
+      pkg.cp_erb("README.md.erb", "README.md")
+      pkg.cp("LICENSE.txt", "LICENSE.txt")
+      pkg.cp_erb("gitignore.erb", ".gitignore")
+
+      case language
+      when :ruby
+        pkg.cp("ruby/Rakefile", "Rakefile")
+        pkg.cp("ruby/Gemfile", "Gemfile")
+        pkg.cp("ruby/.ruby-version", ".ruby-version")
+        pkg.cp_erb("ruby/gemspec.erb", "#{full_project_name}.gemspec")
+        pkg.cp_erb("ruby/#{category}.rb.erb", plugin_path)
+
+      when :java
+        pkg.cp("java/gradle/wrapper/gradle-wrapper.jar", "gradle/wrapper/gradle-wrapper.jar")
+        pkg.cp("java/gradle/wrapper/gradle-wrapper.properties", "gradle/wrapper/gradle-wrapper.properties")
+        pkg.cp("java/gradlew.bat", "gradlew.bat")
+        pkg.cp("java/gradlew", "gradlew")
+        pkg.set_executable("gradlew")
+        pkg.cp_erb("java/build.gradle.erb", "build.gradle")
+        pkg.cp_erb("java/plugin_loader.rb.erb", plugin_path)
+        pkg.cp_erb("java/#{category}.java.erb", "src/main/java/#{java_package_name.gsub(/\./, '/')}/#{java_class_name}.java")
+        pkg.cp_erb("java/test.java.erb", "src/test/java/#{java_package_name.gsub(/\./, '/')}/Test#{java_class_name}.java")
+      end
+
+      extra_guess_erb.each_pair do |erb,dest|
+        pkg.cp_erb(erb, dest)
+      end
+
+      puts ""
+      puts "Plugin template is successfully generated."
+
+      case language
+      when :ruby
+        puts "Next steps:"
+        puts ""
+        puts "  $ cd #{full_project_name}"
+        puts "  $ bundle install                      # install one using rbenv & rbenv-build"
+        puts "  $ bundle exec rake                    # build gem to be released"
+        puts "  $ bundle exec embulk run config.yml   # you can run plugin using this command"
+      when :java
+        puts "Next steps:"
+        puts ""
+        puts "  $ cd #{full_project_name}"
+        puts "  $ ./gradlew package"
+      end
+
+      success = true
+      puts ""
+    ensure
+      FileUtils.rm_rf full_project_name unless success
+    end
+  end
+end