embulk 0.8.18-java → 0.8.19-java

data/lib/embulk/command/embulk_example.rb

@@ -1,33 +0,0 @@
-module Embulk
-  def self.create_example(path)
-    require 'fileutils'
-    require 'zlib'
-
-    puts "  Creating #{path}/"
-    FileUtils.mkdir_p File.join(path, 'csv')
-    puts "  Creating #{path}/csv/"
-
-    puts "  Creating #{path}/csv/sample_01.csv.gz"
-    Zlib::GzipWriter.open(File.join(path, 'csv', 'sample_01.csv.gz')) do |f|
-      f.write <<EOF
-id,account,time,purchase,comment
-1,32864,2015-01-27 19:23:49,20150127,embulk
-2,14824,2015-01-27 19:01:23,20150127,embulk jruby
-3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
-4,11270,2015-01-29 11:54:36,20150129,NULL
-
-EOF
-    end
-
-    puts "  Creating #{path}/seed.yml"
-    File.open(File.join(path, 'seed.yml'), 'w') do |f|
-      f.write <<EOF
-in:
-  type: file
-  path_prefix: "#{File.expand_path File.join(path, 'csv', 'sample_')}"
-out:
-  type: stdout
-EOF
-    end
-  end
-end

data/lib/embulk/command/embulk_generate_bin.rb

@@ -1,62 +0,0 @@
-module Embulk
-  def self.generate_bin(options={})
-    jruby_jar_path = org.jruby.Main.java_class.protection_domain.code_source.location.to_s
-    if __FILE__ =~ /^(?:classpath|uri:classloader):/ || __FILE__.include?('!/')
-      resource_class = org.embulk.command.Runner.java_class
-      ruby_script_path = resource_class.resource("/embulk/command/embulk.rb").to_s
-    else
-      ruby_script_path = File.join(File.dirname(__FILE__), 'embulk.rb')
-    end
-    java_home = java.lang.System.properties['java.home']
-
-    generate_bin_data(jruby_jar_path, ruby_script_path, options.merge(java_home: java_home))
-  end
-
-  def self.generate_bin_data(jruby_jar_path, ruby_script_path, options={})
-    if java_home = options[:java_home]
-      java_home_script = %{export JAVA_HOME='#{java_home}'}
-      java_path = %{"$JAVA_HOME"/bin/java}
-    else
-      java_home_script = %{}
-      java_path = %{java}
-    end
-
-    # TODO parse -D options to set them to java
-    shell_script = <<EOF
-#!/bin/sh
-=begin 2>/dev/null
-#{java_home_script}
-exec #{java_path} -classpath "$0" org.jruby.Main "$0" "$@"
-exit 127
-=end
-EOF
-
-    if options[:bundle_path] == :here
-      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = File.expand_path('..', File.dirname(__FILE__))}
-    elsif path = options[:bundle_path]
-      bundle_path_script = %{ENV['EMBULK_BUNDLE_PATH'] = '#{path}'}
-    else
-      bundle_path_script = b ''
-    end
-
-    ruby_init_script = b <<EOF
-#{bundle_path_script}
-ENV.delete 'GEM_HOME'
-ENV.delete 'GEM_PATH'
-EOF
-
-    ruby_script = b(File.read(ruby_script_path))
-    if i = ruby_script.index(b("\n__END__\n"))
-      # delete contents after __END__
-      ruby_script = ruby_script[0, i]
-    end
-
-    jruby_jar = b(File.read(jruby_jar_path))
-
-    data = shell_script << ruby_init_script + ruby_script << b("\n__END__\n") << jruby_jar
-  end
-
-  def self.b(s)
-    s.force_encoding('ASCII-8BIT')
-  end
-end

data/lib/embulk/command/embulk_migrate_plugin.rb

@@ -1,244 +0,0 @@
-module Embulk
-  def self.migrate_plugin(path)
-    migrator = Migrator.new(path)
-
-    if ms = migrator.match("**/build.gradle", /org\.embulk:embulk-core:([\d\.\+]+)?/)
-      lang = :java
-      from_ver = version(ms[0][1].gsub(/\++/, '0'))   # replaces "0.8.+" to "0.8.0"
-      puts "Detected Java plugin for Embulk #{from_ver}..."
-
-    elsif ms = migrator.match("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s+([\d\.]+)\W+/)
-      lang = :ruby
-      from_ver = version(ms[0][1])
-      puts "Detected Ruby plugin for Embulk #{from_ver}..."
-
-    elsif ms = migrator.match("**/*.gemspec", /embulk-/)
-      lang = :ruby
-      from_ver = version("0.1.0")
-      puts "Detected Ruby plugin for unknown Embulk version..."
-
-    else
-      raise "Failed to detect plugin language and dependency version"
-    end
-
-    case lang
-    when :java
-      migrate_java_plugin(migrator, from_ver)
-    when :ruby
-      migrate_ruby_plugin(migrator, from_ver)
-    end
-
-    if migrator.modified_files.empty?
-      puts "Done. No files are modified."
-    else
-      puts "Done. Please check modifieid files."
-    end
-  end
-
-  def self.migrate_java_plugin(migrator, from_ver)
-    if from_ver < version("0.7.0")
-      # rename CommitReport to TaskReport
-      migrator.replace("**/*.java", /(CommitReport)/, "TaskReport")
-      migrator.replace("**/*.java", /(commitReport)/, "taskReport")
-    end
-
-    # upgrade gradle version
-    if migrator.match("gradle/wrapper/gradle-wrapper.properties", /gradle-[23]\.\d+(\.\d+)?-/)
-      # gradle < 3.2.1
-      require 'embulk/data/package_data'
-      data = PackageData.new("new", migrator.path)
-      migrator.write "gradle/wrapper/gradle-wrapper.properties", data.content("java/gradle/wrapper/gradle-wrapper.properties")
-      migrator.write "gradle/wrapper/gradle-wrapper.jar", data.bincontent("java/gradle/wrapper/gradle-wrapper.jar")
-    end
-
-    # add jsonColumn method
-    if !migrator.match("**/*.java", /void\s+jsonColumn/) && ms = migrator.match("**/*.java", /^(\W+).*?void\s+timestampColumn/)
-      indent = ms.first[1]
-      replace =  <<EOF
-
-#{indent}public void jsonColumn(Column column) {
-#{indent}    throw new UnsupportedOperationException("This plugin doesn't support json type. Please try to upgrade version of the plugin using 'embulk gem update' command. If the latest version still doesn't support json type, please contact plugin developers, or change configuration of input plugin not to use json type.");
-#{indent}}
-
-#{indent}@Override
-EOF
-      migrator.replace("**/*.java", /(\r?\n)(\W+).*?void\s+timestampColumn/, replace)
-    end
-
-    # add sourceCompatibility and targetCompatibility
-    unless migrator.match("build.gradle", /targetCompatibility/)
-      migrator.insert_line("build.gradle", /^([ \t]*)dependencies\s*{/) {|m|
-        "#{m[1]}targetCompatibility = 1.7\n"
-      }
-    end
-    unless migrator.match("build.gradle", /sourceCompatibility/)
-      migrator.insert_line("build.gradle", /^([ \t]*)targetCompatibility/) {|m|
-        "#{m[1]}sourceCompatibility = 1.7"
-      }
-    end
-
-    # add checkstyle
-    unless migrator.match("build.gradle", /id\s+(?<quote>["'])checkstyle\k<quote>/)
-      migrator.insert_line("build.gradle", /^([ \t]*)id( +)(["'])java["']/) {|m|
-        "#{m[1]}id#{m[2]}#{m[3]}checkstyle#{m[3]}"
-      }
-      migrator.write "config/checkstyle/checkstyle.xml", migrator.new_data.content("java/config/checkstyle/checkstyle.xml")
-    end
-
-    unless migrator.match("build.gradle", /checkstyle\s+{/)
-      migrator.write "config/checkstyle/default.xml", migrator.new_data.content("java/config/checkstyle/default.xml")
-      migrator.insert_line("build.gradle", /^([ \t]*)task\s+gem\W.*{/) {|m|
-        <<-EOF
-#{m[1]}checkstyle {
-#{m[1]}    configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
-#{m[1]}    toolVersion = '6.14.1'
-#{m[1]}}
-#{m[1]}checkstyleMain {
-#{m[1]}    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-#{m[1]}    ignoreFailures = true
-#{m[1]}}
-#{m[1]}checkstyleTest {
-#{m[1]}    configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-#{m[1]}    ignoreFailures = true
-#{m[1]}}
-#{m[1]}task checkstyle(type: Checkstyle) {
-#{m[1]}    classpath = sourceSets.main.output + sourceSets.test.output
-#{m[1]}    source = sourceSets.main.allJava + sourceSets.test.allJava
-#{m[1]}}
-EOF
-      }
-    end
-
-    #
-    # add rules...
-    ##
-
-    # update version at the end
-    migrator.replace("**/build.gradle", /org\.embulk:embulk-(?:core|standards):([\d\.\+]+)?/, Embulk::VERSION)
-  end
-
-  def self.migrate_ruby_plugin(migrator, from_ver)
-    #
-    # add rules...
-    ##
-
-    migrator.write(".ruby-version", "jruby-9.1.5.0")
-
-    # update version at the end
-    if from_ver <= version("0.1.0")
-      # add add_development_dependency
-      migrator.insert_line("**/*.gemspec", /([ \t]*\w+)\.add_development_dependency/) {|m|
-        "#{m[1]}.add_development_dependency 'embulk', ['>= #{Embulk::VERSION}']"
-      }
-    else
-      unless migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s*(\~\>\s*[\d\.]+)\W+/, ">= #{Embulk::VERSION}")
-        migrator.replace("**/*.gemspec", /add_(?:development_)?dependency\s+\W+embulk\W+\s*([\d\.]+)\W+/, Embulk::VERSION)
-      end
-    end
-  end
-
-  private
-
-  def self.version(str)
-    Gem::Version.new(str)
-  end
-
-  class Migrator
-    def initialize(path)
-      @path = path
-      @modified_files = {}
-
-      require 'fileutils'
-      require 'embulk/data/package_data'
-      @new_data = PackageData.new("new", path)
-    end
-
-    attr_reader :path
-    attr_reader :new_data
-
-    def modified_files
-      @modified_files.keys
-    end
-
-    def match(glob, pattern)
-      ms = Dir[File.join(@path, glob)].map do |file|
-        read(file).match(pattern)
-      end.compact
-      return nil if ms.empty?
-      ms
-    end
-
-    def replace(glob, pattern, text=nil)
-      ms = Dir[File.join(@path, glob)].map do |file|
-        data = read(file)
-        first = nil
-        pos = 0
-        while pos < data.length
-          m = data.match(pattern, pos)
-          break unless m
-          first ||= m
-          replace = text || yield(m)
-          data = m.pre_match + data[m.begin(0)..(m.begin(1)-1)] + replace + data[m.end(1)..(m.end(0)-1)] + m.post_match
-          pos = m.begin(1) + replace.length + (m.end(0) - m.end(1))
-        end
-        if first
-          modify(file, data)
-        end
-        first
-      end.compact
-      return nil if ms.empty?
-      ms
-    end
-
-    def insert_line(glob, pattern, text: nil)
-      ms = Dir[File.join(@path, glob)].map do |file|
-        data = read(file)
-        if m = data.match(pattern)
-          ln = m.pre_match.split("\n").count
-          replace = text || yield(m)
-          lines = data.split("\n", -1)  # preserve the last empty line
-          lines.insert(ln + 1, replace)
-          data = lines.join("\n")
-          modify(file, data)
-          m
-        end
-      end.compact
-      return nil if ms.empty?
-      ms
-    end
-
-    def data(file)
-      read(File.join(@path, file))
-    end
-
-    def write(file, data)
-      dst = File.join(@path, file)
-      FileUtils.mkdir_p File.dirname(dst)
-      modify(dst, data)
-    end
-
-    private
-
-    def modify(path, data)
-      orig = read(path) rescue nil
-      if orig != data
-        File.write(path, data)
-        unless @modified_files.has_key?(path)
-          if orig
-            puts "  Modified #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
-          else
-            puts "  Created #{path.sub(/^#{Regexp.escape(@path)}/, '')}"
-          end
-          @modified_files[path] = true
-        end
-      end
-      nil
-    end
-
-    def read(path)
-      # external_encoding: assumes source code is written in UTF-8.
-      # internal_encoding: process files using UTF-8 so that modified data (data written to the file) becomes UTF-8.
-      File.read(path, external_encoding: 'UTF-8', internal_encoding: 'UTF-8')
-    end
-  end
-end

data/lib/embulk/command/embulk_new_plugin.rb

@@ -1,126 +0,0 @@
-module Embulk
-  def self.new_plugin(name, language, category)
-    require 'embulk/data/package_data'
-    require 'embulk/version'
-    require 'fileutils'
-
-    embulk_category = category
-    embulk_category = :input if category == :file_input
-    embulk_category = :output if category == :file_output
-
-    name = name.gsub(/[^a-zA-Z0-9_]+/, '_')  # replace '-' to '_'
-
-    full_project_name = "embulk-#{embulk_category}-#{name}"
-    plugin_dir = "lib/embulk"
-    plugin_path = "#{plugin_dir}/#{embulk_category}/#{name}.rb"
-
-    if File.exist?(full_project_name)
-      raise "./#{full_project_name} already exists. Please delete it first."
-    end
-    FileUtils.mkdir_p(full_project_name)
-
-    puts "Creating #{full_project_name}/"
-
-    success = false
-    begin
-      #
-      # Generate gemspec
-      #
-      author = `git config user.name`.strip rescue ""
-      author = "YOUR_NAME" if author.empty?
-      email = `git config user.email`.strip rescue ""
-      email = "YOUR_NAME" if email.empty?
-
-      # variables used in erb templates
-      ruby_class_name = name.split('_').map {|a| a.capitalize }.join
-      java_iface_name = category.to_s.split('_').map {|a| a.capitalize }.join
-      java_class_name = name.split('_').map {|a| a.capitalize }.join + java_iface_name + "Plugin"
-      java_package_name = "org.embulk.#{embulk_category}.#{name}"
-      display_name = name.split('_').map {|a| a.capitalize }.join(' ')
-      display_category = category.to_s.gsub('_', ' ')
-
-      extra_guess_erb = {}
-
-      case category
-      when :input
-        description = %[Loads records from #{display_name}.]
-      when :file_input
-        description = %[Reads files stored on #{display_name}.]
-      when :parser
-        description = %[Parses #{display_name} files read by other file input plugins.]
-        extra_guess_erb["ruby/parser_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
-      when :decoder
-        description = %[Decodes #{display_name}-encoded files read by other file input plugins.]
-        extra_guess_erb["ruby/decoder_guess.rb.erb"] = "#{plugin_dir}/guess/#{name}.rb"
-      when :output
-        description = %[Dumps records to #{display_name}.]
-      when :file_output
-        description = %[Stores files on #{display_name}.]
-      when :formatter
-        description = %[Formats #{display_name} files for other file output plugins.]
-      when :encoder
-        description = %[Encodes files using #{display_name} for other file output plugins.]
-      when :filter
-        description = %[#{display_name}]
-      end
-
-      #
-      # Generate project repository
-      #
-      pkg = Embulk::PackageData.new("new", full_project_name, binding())
-
-      pkg.cp_erb("README.md.erb", "README.md")
-      pkg.cp("LICENSE.txt", "LICENSE.txt")
-      pkg.cp_erb("gitignore.erb", ".gitignore")
-
-      case language
-      when :ruby
-        pkg.cp("ruby/Rakefile", "Rakefile")
-        pkg.cp("ruby/Gemfile", "Gemfile")
-        pkg.cp("ruby/.ruby-version", ".ruby-version")
-        pkg.cp_erb("ruby/gemspec.erb", "#{full_project_name}.gemspec")
-        pkg.cp_erb("ruby/#{category}.rb.erb", plugin_path)
-
-      when :java
-        pkg.cp("java/gradle/wrapper/gradle-wrapper.jar", "gradle/wrapper/gradle-wrapper.jar")
-        pkg.cp("java/gradle/wrapper/gradle-wrapper.properties", "gradle/wrapper/gradle-wrapper.properties")
-        pkg.cp("java/gradlew.bat", "gradlew.bat")
-        pkg.cp("java/gradlew", "gradlew")
-        pkg.set_executable("gradlew")
-        pkg.cp("java/config/checkstyle/checkstyle.xml","config/checkstyle/checkstyle.xml")
-        pkg.cp("java/config/checkstyle/default.xml","config/checkstyle/default.xml")
-        pkg.cp_erb("java/build.gradle.erb", "build.gradle")
-        pkg.cp_erb("java/plugin_loader.rb.erb", plugin_path)
-        pkg.cp_erb("java/#{category}.java.erb", "src/main/java/#{java_package_name.gsub(/\./, '/')}/#{java_class_name}.java")
-        pkg.cp_erb("java/test.java.erb", "src/test/java/#{java_package_name.gsub(/\./, '/')}/Test#{java_class_name}.java")
-      end
-
-      extra_guess_erb.each_pair do |erb,dest|
-        pkg.cp_erb(erb, dest)
-      end
-
-      puts ""
-      puts "Plugin template is successfully generated."
-
-      case language
-      when :ruby
-        puts "Next steps:"
-        puts ""
-        puts "  $ cd #{full_project_name}"
-        puts "  $ bundle install                      # install one using rbenv & rbenv-build"
-        puts "  $ bundle exec rake                    # build gem to be released"
-        puts "  $ bundle exec embulk run config.yml   # you can run plugin using this command"
-      when :java
-        puts "Next steps:"
-        puts ""
-        puts "  $ cd #{full_project_name}"
-        puts "  $ ./gradlew package"
-      end
-
-      success = true
-      puts ""
-    ensure
-      FileUtils.rm_rf full_project_name unless success
-    end
-  end
-end