embulk 0.8.27-java → 0.8.28-java

data/embulk-core/src/main/java/org/embulk/EmbulkSetup.java

@@ -0,0 +1,70 @@
+package org.embulk;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.HashMap;
+import java.util.Map;
+import org.embulk.config.ConfigSource;
+import org.jruby.embed.LocalContextScope;
+import org.jruby.embed.LocalVariableBehavior;
+import org.jruby.embed.ScriptingContainer;
+
+/**
+ * EmbulkSetup initiates an EmbulkRunner instance. It was originally implemented with Ruby in lib/embulk.rb.
+ *
+ * NOTE: Developers should not depend on this EmbulkSetup class. This class is created tentatively, and may be
+ * re-implemented again in a different style.
+ */
+public class EmbulkSetup
+{
+    @Deprecated
+    public static EmbulkRunner setupWithNewScriptingContainer(final Map<String, Object> systemConfigGiven)
+    {
+        // The JRuby instance is a global singleton so that the settings here affects later execution.
+        // The local variable should be persistent so that local variables are set through ScriptingContainer.put.
+        final ScriptingContainer globalJRubyContainer =
+            new ScriptingContainer(LocalContextScope.SINGLETON, LocalVariableBehavior.PERSISTENT);
+        return setup(systemConfigGiven, globalJRubyContainer);
+    }
+
+    public static EmbulkRunner setup(
+            final Map<String, Object> systemConfigGiven,
+            final ScriptingContainer globalJRubyContainer)
+    {
+        // NOTE: When it was in Ruby "require 'json'" was required to format the system config into a JSON string.
+
+        // NOTE: require_classpath is called only when Embulk is loaded as a Ruby gem. (lib/embulk.rb)
+
+        final HashMap<String, Object> systemConfigModified = new HashMap<String, Object>(systemConfigGiven);
+
+        // use the global ruby runtime for all ScriptingContainer
+        // injected by org.embulk.jruby.JRubyScriptingModule
+        systemConfigModified.put("use_global_ruby_runtime", true);
+
+        // Calling ObjectMapper simply just as a formatter here so that it does not depend much on Jackson.
+        // Do not leak the Jackson object outside.
+        final ObjectMapper jacksonObjectMapper = new ObjectMapper();
+        final String systemConfigJson;
+        try {
+            systemConfigJson = jacksonObjectMapper.writeValueAsString(systemConfigModified);
+        }
+        catch (JsonProcessingException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        final EmbulkEmbed.Bootstrap bootstrap = new org.embulk.EmbulkEmbed.Bootstrap();
+        final ConfigSource systemConfig = bootstrap.getSystemConfigLoader().fromJsonString(systemConfigJson);
+        bootstrap.setSystemConfig(systemConfig);
+        final EmbulkEmbed embed = bootstrap.initialize();  // see embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.
+
+        // see also embulk/java/bootstrap.rb loaded by JRubyScriptingModule
+        globalJRubyContainer.runScriptlet("module Embulk; end");
+        globalJRubyContainer.put("__internal_embulk_setup_embed__", embed);
+        globalJRubyContainer.put("__internal_embulk_setup_global_jruby_container__", globalJRubyContainer);
+        globalJRubyContainer.runScriptlet("Embulk.const_set :Runner, Embulk::EmbulkRunner.new(__internal_embulk_setup_embed__)");
+        globalJRubyContainer.remove("__internal_embulk_setup_global_jruby_container__");
+        globalJRubyContainer.remove("__internal_embulk_setup_embed__");
+
+        return new EmbulkRunner(embed, globalJRubyContainer);
+    }
+}

data/embulk-core/src/main/java/org/embulk/plugin/PluginClassLoader.java

@@ -49,7 +49,7 @@ public class PluginClassLoader
      *
      * Some plugins (embulk-input-jdbc, for example) are calling this method to load external JAR files.
      *
-     * @see https://github.com/embulk/embulk-input-jdbc/blob/ebfff0b249d507fc730c87e08b56e6aa492060ca/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java#L586-L595
+     * @see <a href="https://github.com/embulk/embulk-input-jdbc/blob/ebfff0b249d507fc730c87e08b56e6aa492060ca/embulk-input-jdbc/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java#L586-L595">embulk-input-jdbc</a>
      */
     public void addPath(Path path)
     {
@@ -84,8 +84,8 @@ public class PluginClassLoader
      *
      * If the specified class is "NOT parent-first", the 2nd and 3rd actions are swapped.
      *
-     * @see https://docs.oracle.com/javase/7/docs/api/java/lang/ClassLoader.html#loadClass(java.lang.String,%20boolean)
-     * @see http://hg.openjdk.java.net/jdk7u/jdk7u/jdk/file/jdk7u141-b02/src/share/classes/java/lang/ClassLoader.java
+     * @see <a href="https://docs.oracle.com/javase/7/docs/api/java/lang/ClassLoader.html#loadClass(java.lang.String,%20boolean)">Oracle Java7's ClassLoader#loadClass</a>
+     * @see <a href="http://hg.openjdk.java.net/jdk7u/jdk7u/jdk/file/jdk7u141-b02/src/share/classes/java/lang/ClassLoader.java">OpenJDK7's ClassLoader</a>
      */
     @Override
     protected Class<?> loadClass(String name, boolean resolve)

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.28
     release/release-0.8.27
     release/release-0.8.26
     release/release-0.8.25

data/embulk-docs/src/release/release-0.8.28.rst

@@ -0,0 +1,14 @@
+Release 0.8.28
+==================================
+
+General Changes
+------------------
+
+* Make selfrun scripts not to run JRuby to get ruby_version. [#721]
+* Un-JRuby the bootstrap. [#706] [#708] [#543] [#725] [#726] [#727] [#729] [#731] [#741] [#742]
+* Fix the "migrate" subcommand. [#732]
+* Improve building. [#716] [#724] [#728]
+
+Release Date
+------------------
+2017-07-27

data/lib/embulk.rb

@@ -17,7 +17,7 @@ module Embulk
     end
   end
 
-  def self.require_classpath
+  def self.require_classpath(already_warned=false)
     if __FILE__.include?("!")
       # single jar. __FILE__ should point path/to/embulk.jar!/embulk.rb
       # which means that embulk.jar is already loaded in this JVM.
@@ -28,6 +28,16 @@ module Embulk
     else
       # gem package. __FILE__ should point path/to/embulk/lib/embulk.rb
       # that requires here to load ../classpath/*.jar to start EmbulkEmbed.
+
+      unless already_warned
+        STDERR.puts "################################################################################"
+        STDERR.puts "[WARN] Embulk's gem package is deprecated, and will be removed from v0.9."
+        STDERR.puts "[WARN] Use the jar version installed from http://dl.embulk.org/ instead."
+        STDERR.puts "[WARN] See the issue and comment at: https://github.com/embulk/embulk/issues/628"
+        STDERR.puts "################################################################################"
+        STDERR.puts ""
+      end
+
       gem_root = File.expand_path('..', File.dirname(__FILE__))
       classpath_dir = File.join(gem_root, "classpath")
       jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort
@@ -38,27 +48,21 @@ module Embulk
   end
 
   def self.setup(system_config={})
+    STDERR.puts "################################################################################"
+    STDERR.puts "[WARN] Embulk's gem package is deprecated, and will be removed from v0.9."
+    STDERR.puts "[WARN] Use the jar version installed from http://dl.embulk.org/ instead."
+    STDERR.puts "[WARN] See the issue and comment at: https://github.com/embulk/embulk/issues/628"
+    STDERR.puts "################################################################################"
+    STDERR.puts ""
+
     unless RUBY_PLATFORM =~ /java/i
       raise "Embulk.setup works only with JRuby."
     end
 
-    require 'json'
-
-    require_classpath
-
-    systemConfigJson = system_config.merge({
-      # use the global ruby runtime for all ScriptingContainer
-      # injected by org.embulk.jruby.JRubyScriptingModule
-      use_global_ruby_runtime: true
-    }).to_json
-
-    bootstrap = org.embulk.EmbulkEmbed::Bootstrap.new
-    systemConfig = bootstrap.getSystemConfigLoader.fromJsonString(systemConfigJson)
-    bootstrap.setSystemConfig(systemConfig)
-    embed = bootstrap.java_method(:initialize).call  # see embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.
+    # require 'json' -- was required to format the system config into a JSON string.
 
-    # see also embulk/java/bootstrap.rb loaded by JRubyScriptingModule
+    require_classpath(true)
 
-    Embulk.const_set :Runner, EmbulkRunner.new(embed)
+    Java::org.embulk.EmbulkSetup::setupWithNewScriptingContainer(Java::java.util.HashMap.new(system_config))
   end
 end

data/lib/embulk/runner.rb

@@ -4,195 +4,64 @@ module Embulk
   # Runner = EmbulkRunner.new
 
   class EmbulkRunner
-    def initialize(embed)
+    def initialize(embed, jruby_container=nil)
       @embed = embed  # org.embulk.EmbulkEmbed
+      if jruby_container
+        @jruby_container = jruby_container
+      else
+        # The ScriptingContainer instance below is the same JRuby instance with the one running this runner.rb itself
+        # as it's LocalContextScope.SINGLETON.
+        @jruby_container = Java::org.jruby.embed.ScriptingContainer.new(
+          Java::org.jruby.embed.LocalContextScope::SINGLETON, Java::org.jruby.embed.LocalVariableBehavior::PERSISTENT)
+      end
     end
 
     def guess(config, options={})
-      configSource = read_config(config, options)
-      output_path = options[:next_config_output_path]
-
-      check_file_writable(output_path)
+      output_path =
+        (options[:next_config_output_path] ? Java::java.lang.String.new(options[:next_config_output_path]) : nil)
+      embulk_runner_java = Java::org.embulk.EmbulkRunner.new(@embed, @jruby_container)
 
-      configDiff = @embed.guess(configSource)
-
-      guessedConfigSource = configSource.merge(configDiff)
-      yaml = write_config(output_path, guessedConfigSource)
-      STDERR.puts yaml
-      if output_path
-        puts "Created '#{output_path}' file."
-      else
-        puts "Use -o PATH option to write the guessed config file to a file."
+      case config
+      when String
+        embulk_runner_java.guess(Java::java.lang.String.new(config), output_path)
+      when Hash, DataSource
+        embulk_runner_java.guess(DataSource.new(config).to_java, output_path)
       end
 
       nil
     end
 
     def preview(config, options={})
-      configSource = read_config(config, options)
-      format = options[:format]
-
-      previewResult = @embed.preview(configSource)
+      format = (options[:format] ? Java::java.lang.String.new(options[:format]) : nil)
+      embulk_runner_java = Java::org.embulk.EmbulkRunner.new(@embed, @jruby_container)
 
-      modelManager = @embed.getModelManager
-      printer =
-        case format || "table"
-        when "table"
-          org.embulk.command.TablePreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
-        when "vertical"
-          org.embulk.command.VerticalPreviewPrinter.new(java.lang.System.out, modelManager, previewResult.getSchema)
-        else
-          raise ArgumentError, "Unknown preview output format '#{format}'. Supported formats: table, vertical"
-        end
-
-      printer.printAllPages(previewResult.getPages)
-      printer.finish
+      case config
+      when String
+        embulk_runner_java.preview(Java::java.lang.String.new(config), format)
+      when Hash, DataSource
+        embulk_runner_java.preview(DataSource.new(config).to_java, format)
+      end
 
       nil
     end
 
     def run(config, options={})
-      configSource = read_config(config, options)
-      config_diff_path = options[:next_config_diff_path]
-      output_path = options[:next_config_output_path]  # deprecated
-      resume_state_path = options[:resume_state_path]
-
-      check_file_writable(output_path)  # deprecated
-      check_file_writable(config_diff_path)
-      check_file_writable(resume_state_path)
-
-      if config_diff_path && File.size(config_diff_path) > 0
-        lastConfigDiff = read_config(config_diff_path, options)
-        configSource = configSource.merge(lastConfigDiff)
-      end
-
-      if resume_state_path
-        begin
-          resumeConfig = read_yaml_config_file(resume_state_path)
-          resumeConfig = nil if resumeConfig.isEmpty
-        rescue
-          # TODO log?
-          resumeConfig = nil
-        end
-      end
-
-      if resumeConfig
-        resumableResult = @embed.resumeState(configSource, resumeConfig).resume
-      elsif resume_state_path
-        resumableResult = @embed.runResumable(configSource)
-      else
-        executionResult = @embed.run(configSource)
-      end
-
-      unless executionResult
-        unless resumableResult.isSuccessful
-          if resumableResult.getTransactionStage.isBefore(org.embulk.exec.TransactionStage::RUN)
-            # retry without resume state file if no tasks started yet
-            # delete resume file
-            File.delete(resume_state_path) rescue nil if resume_state_path
-          else
-            Embulk.logger.info "Writing resume state to '#{resume_state_path}'"
-            write_config(resume_state_path, resumableResult.getResumeState)
-            Embulk.logger.info "Resume state is written. Run the transaction again with -r option to resume or use \"cleanup\" subcommand to delete intermediate data."
-          end
-          raise resumableResult.getCause
-        end
-        executionResult = resumableResult.getSuccessfulResult
-      end
+      config_diff_path =
+        (options[:next_config_diff_path] ? Java::java.lang.String.new(options[:next_config_diff_path]) : nil)
+      output_path =  # deprecated
+        (options[:next_config_output_path] ? Java::java.lang.String.new(options[:next_config_output_path]) : nil)
+      resume_state_path =
+        (options[:resume_state_path] ? Java::java.lang.String.new(options[:resume_state_path]) : nil)
+      embulk_runner_java = Java::org.embulk.EmbulkRunner.new(@embed, @jruby_container)
 
-      # delete resume file
-      File.delete(resume_state_path) rescue nil if resume_state_path
-
-      configDiff = executionResult.getConfigDiff
-      Embulk.logger.info("Committed.")
-      Embulk.logger.info("Next config diff: #{configDiff.toString}")
-
-      write_config(config_diff_path, configDiff)
-      write_config(output_path, configSource.merge(configDiff))  # deprecated
-    end
-
-    #def resume_state(config, options={})
-    #  configSource = read_config(config, options)
-    #  Resumed.new(self, DataSource.from_java(configSource), options)
-    #end
-
-    private
-
-    def read_config(config, options={})
       case config
       when String
-        case config
-        when /\.ya?ml\.liquid$/
-          require 'liquid'
-          template_params = options[:template_params] || {}
-          template_include_path = File.expand_path(options[:template_include_path] || File.dirname(config)) unless options[:template_include_path] == false
-          @embed.newConfigLoader.fromYamlString run_liquid(File.read(config), template_params, template_include_path)
-        when /\.ya?ml$/
-          @embed.newConfigLoader.fromYamlString File.read(config)
-        else
-          raise ConfigError.new("Unsupported file extension. Supported file extensions are .yml and .yml.liquid: #{config}")
-        end
-
+        embulk_runner_java.run(Java::java.lang.String.new(config), config_diff_path, output_path, resume_state_path)
       when Hash, DataSource
-        DataSource.new(config).to_java
+        embulk_runner_java.run(DataSource.new(config).to_java, config_diff_path, output_path, resume_state_path)
       end
-    end
-
-    def read_yaml_config_file(path)
-      @embed.newConfigLoader.fromYamlString File.read(config)
-    end
-
-    def run_liquid(source, params, template_include_path)
-      require 'liquid'
-      template = Liquid::Template.parse(source, :error_mode => :strict)
-      template.registers[:file_system] = Liquid::LocalFileSystem.new(template_include_path, "_%s.yml.liquid") if template_include_path
-
-      data = {
-        "env" => ENV.to_h,
-      }.merge(params)
 
-      template.render(data)
-    end
-
-    def check_file_writable(path)
-      if path
-        # Open file with append mode and do nothing.
-        # If file is not writable, it throws an exception.
-        File.open(path, "ab")
-      end
-    end
-
-    def write_config(path, modelObject)
-      yaml = dump_yaml(modelObject)
-      if path
-        File.write(path, yaml)
-      end
-      yaml
-    end
-
-    def dump_yaml(modelObject)
-      modelManager = @embed.getModelManager
-      obj = modelManager.readObject(java.lang.Object.java_class, modelManager.writeObject(modelObject))
-      return org.yaml.snakeyaml.Yaml.new.dump(obj)
+      nil
     end
-
-    #class Runnable
-    #  def initialize(runner, config, options)
-    #    @runner = runner
-    #    @config = config
-    #    @options = options
-    #  end
-    #
-    #  attr_reader :config
-    #
-    #  def preview(options={})
-    #    @runner.preview(@config, @options.merge(options))
-    #  end
-    #
-    #  def run(options={})
-    #    @runner.run(@config, @options.merge(options))
-    #  end
-    #end
   end
-
 end

data/lib/embulk/version.rb

@@ -3,7 +3,7 @@
 module Embulk
   @@warned = false
 
-  VERSION_INTERNAL = '0.8.27'
+  VERSION_INTERNAL = '0.8.28'
 
   DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
   def self.const_missing(name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.27
+  version: 0.8.28
 platform: java
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-07-10 00:00:00.000000000 Z
+date: 2017-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,14 +150,15 @@ files:
 - classpath/bval-core-0.5.jar
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
+- classpath/commons-cli-1.3.1.jar
 - classpath/commons-collections-3.2.1.jar
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang-2.4.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-cli-0.8.27.jar
-- classpath/embulk-core-0.8.27.jar
-- classpath/embulk-jruby-strptime-0.8.27.jar
-- classpath/embulk-standards-0.8.27.jar
+- classpath/embulk-cli-0.8.28.jar
+- classpath/embulk-core-0.8.28.jar
+- classpath/embulk-jruby-strptime-0.8.28.jar
+- classpath/embulk-standards-0.8.28.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -191,18 +192,37 @@ files:
 - classpath/validation-api-1.1.0.Final.jar
 - classpath/velocity-1.7.jar
 - embulk-cli/build.gradle
-- embulk-cli/src/main/bat/selfrun.bat
+- embulk-cli/src/main/bat/selfrun.bat.template
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkBundle.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLineException.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkExample.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkGlobalJRubyScriptingContainer.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkNew.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkSelfUpdate.java
+- embulk-cli/src/main/java/org/embulk/cli/EmbulkSubcommand.java
 - embulk-cli/src/main/java/org/embulk/cli/Main.java
-- embulk-cli/src/main/sh/selfrun.sh
+- embulk-cli/src/main/java/org/embulk/cli/parse/AbstractHelpLineDefinition.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/CliHelpFormatterWithHelpMessages.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/CliOptionsWithHelpMessages.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineHelpRequired.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParseException.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParser.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageAsCliOption.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageLineDefinition.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/OptionBehavior.java
+- embulk-cli/src/main/java/org/embulk/cli/parse/OptionDefinition.java
+- embulk-cli/src/main/sh/selfrun.sh.template
 - embulk-cli/src/test/java/org/embulk/cli/DummyMain.java
 - embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java
 - embulk-core/build.gradle
 - embulk-core/src/main/java/org/embulk/EmbulkEmbed.java
+- embulk-core/src/main/java/org/embulk/EmbulkRunner.java
 - embulk-core/src/main/java/org/embulk/EmbulkService.java
+- embulk-core/src/main/java/org/embulk/EmbulkSetup.java
 - embulk-core/src/main/java/org/embulk/EmbulkVersion.java
 - embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java
 - embulk-core/src/main/java/org/embulk/command/TablePreviewPrinter.java
@@ -537,6 +557,7 @@ files:
 - embulk-docs/src/release/release-0.8.25.rst
 - embulk-docs/src/release/release-0.8.26.rst
 - embulk-docs/src/release/release-0.8.27.rst
+- embulk-docs/src/release/release-0.8.28.rst
 - embulk-docs/src/release/release-0.8.3.rst
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst
@@ -688,9 +709,6 @@ files:
 - lib/embulk.rb
 - lib/embulk/buffer.rb
 - lib/embulk/column.rb
-- lib/embulk/command/embulk_bundle.rb
-- lib/embulk/command/embulk_main.rb
-- lib/embulk/command/embulk_run.rb
 - lib/embulk/data/bundle/.bundle/config
 - lib/embulk/data/bundle/.ruby-version
 - lib/embulk/data/bundle/Gemfile