embulk 0.8.32-java → 0.8.33-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2df25ebf0f892234350ee64d8ed0b2d0d09fc1a2
-  data.tar.gz: e2b776f7de4f56904dce422fb3422ff221cca124
+  metadata.gz: fd96601a8c321a46fc090e7072d14fd0a529e1b5
+  data.tar.gz: a9ce23a6473b3cfa3812231fd45cc7ca9869c7d2
 SHA512:
-  metadata.gz: 927d56589043664d0064a0d4c4996735325daaeef47de6979f15076961c8eaf9a0acb3a28c5ccd6ecbc635ca88acb32028b57cb531b6728dc594bd7d3318655e
-  data.tar.gz: 47adef5189e27a8810aec2c9d4bd4c6228d55391e9d300ca1cdb3b4233e1e7b180c68acc9a4dc5d500137006a7ad9d991ea57062161c05783d5402f393641e9d
+  metadata.gz: b7068bb0463ac5d8e6f44bd9fa3bd564a83d4efe835a12eca9b18a4d7964d66f9dfb57efa2b3e8a453bf243c5956de28da7ccd010c0dcc9d1551ae2703d77200
+  data.tar.gz: 8d103a47eaec288a1df2036aa48437fe7f2d88f2cea4746d05684510bc688e346f1bd2969298332b8bccabdc22f5e0d483104b70dcd5ba5833e85961baa8e60e

data/bin/embulk

@@ -15,34 +15,6 @@ if RUBY_PLATFORM =~ /java/i
     # bin/embulk is started by CRuby (embulk gem for CRuby is installed), and re-run by JRuby at the end of the script.
     # Bundler may be enabled in this path.
     ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
-
-    # Handle environment variables before initializing the global JRuby instance in org.embulk.EmbulkRunner.
-    # See: https://github.com/embulk/embulk/pull/604
-
-    bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
-    bundle_path = nil if bundle_path.empty?
-
-    # The -b or --bundle option are processed in org.embulk.EmbulkRunner. The option should be kept.
-    bundle_option_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
-    if bundle_option_index
-      bundle_path = ARGV[bundle_option_index + 1]
-    end
-
-    if bundle_path
-      ENV['EMBULK_BUNDLE_PATH'] = bundle_path
-      ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
-      ENV.delete('GEM_HOME')
-      ENV.delete('GEM_PATH')
-    else
-      ENV.delete('EMBULK_BUNDLE_PATH')
-      user_home = java.lang.System.properties["user.home"] || ENV['HOME']
-      unless user_home
-        raise "HOME environment variable is not set."
-      end
-      ENV['GEM_HOME'] = File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
-      ENV['GEM_PATH'] = ''
-      ENV.delete('BUNDLE_GEMFILE')
-    end
   else
     # bin/embulk is started by JRuby (embulk gem for JRuby is installed).
     # Bundler must be disabled in this path not to bother the JRuby's bundler.
@@ -76,6 +48,11 @@ overwrite_optimize = nil
 until ARGV.empty?
   v = ARGV[0]
   case v
+  when /-E(.*)/
+    external_script = v[2..-1]
+    STDERR.puts "Running an external Ruby script: #{external_script}"
+    load external_script
+    ARGV.shift
   when "-J+O"
     overwrite_optimize = true
     ARGV.shift

data/build.gradle

@@ -24,7 +24,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.32'
+    version = '0.8.33'
 
     ext {
         jrubyVersion = '9.1.5.0'
@@ -217,27 +217,18 @@ task classpath(dependsOn: ['build', ':embulk-cli:classpath']) { doLast {} }
 
 clean { delete 'classpath' }
 
-//
-// Gets RbConfig::CONFIG['ruby_version'] from the same version of JRuby with Embulk's embedded.
-//
-task retrieveRubyVersion(type: JRubyExec) {
-    standardOutput = new ByteArrayOutputStream()
-    jrubyArgs '-e', "print RbConfig::CONFIG['ruby_version']"
-}
-
 //
 // cli task
 //
-task cli(dependsOn: ['retrieveRubyVersion', ':embulk-cli:shadowJar']) {
+task cli(dependsOn: ':embulk-cli:shadowJar') {
     doLast {
-        def rubyVersion = retrieveRubyVersion.standardOutput.toString().trim()
         file('pkg').mkdirs()
         File f = file("pkg/embulk-${project.version}.jar")
         f.write("")
         f.append("\n: <<END_OF_EMBULK_SELFRUN_BATCH_PART\r\n")
-        f.append(file("embulk-cli/src/main/bat/selfrun.bat.template").getText().replace("<%= ruby_version %>", rubyVersion))
+        f.append(file("embulk-cli/src/main/bat/selfrun.bat").readBytes())
         f.append("\r\nEND_OF_EMBULK_SELFRUN_BATCH_PART\r\n\n")
-        f.append(file("embulk-cli/src/main/sh/selfrun.sh.template").getText().replace("<%= ruby_version %>", rubyVersion))
+        f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
         f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
         f.setExecutable(true)
     }

data/embulk-cli/src/main/bat/{selfrun.bat.template → selfrun.bat}

@@ -33,59 +33,6 @@ if "%overwrite_optimize%" == "true" (
     )
 )
 
-setlocal enabledelayedexpansion
-
-set found_bundle_option=0
-
-for %%a in (%*) do (
-    if %%a == -b (
-        set found_bundle_option=1
-    ) else if %%a == --bundle (
-        set found_bundle_option=1
-    ) else if !found_bundle_option! == 1 (
-        set embulk_bundle_path=%%a
-        set found_bundle_option=2
-    )
-)
-
-endlocal && set EMBULK_BUNDLE_PATH=%embulk_bundle_path%
-
-if not defined EMBULK_BUNDLE_PATH (
-    set EMBULK_BUNDLE_PATH=
-    set GEM_PATH=""
-) else (
-    if not exist "%EMBULK_BUNDLE_PATH%\" (
-        echo Directory not found: "%EMBULK_BUNDLE_PATH%"
-        exit /b 1
-    )
-    set GEM_PATH=
-)
-
-setlocal enabledelayedexpansion
-
-if not defined EMBULK_BUNDLE_PATH (
-    set bundle_gemfile=
-) else (
-    call :get_absolute_path %EMBULK_BUNDLE_PATH%
-    set bundle_gemfile=!absolute_path!\Gemfile
-    if not exist !bundle_gemfile! (
-        echo Gemfile not found: "!bundle_gemfile!"
-        exit /b 1
-    )
-)
-
-endlocal && set BUNDLE_GEMFILE=%bundle_gemfile%
-
-setlocal enabledelayedexpansion
-
-if not defined EMBULK_BUNDLE_PATH (
-    set gem_home=%USERPROFILE%\.embulk\jruby\<%= ruby_version %>
-) else (
-    set gem_home=
-)
-
-endlocal && set GEM_HOME=%gem_home%
-
 if "%optimize%" == "true" (
     set java_args=-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC %java_args%
 ) else (
@@ -114,6 +61,12 @@ if "%status%" == "rest" (
 ) else if "%status%" == "read" (
     call :read_file %arg%
 
+) else if "%arg:~0,2%" == "-E" (
+    if not "%arg:~2%" == "" (
+        echo Running an external batch file: %arg:~2%
+        call %arg:~2%
+    )
+
 ) else if "%arg%" == "-J+O" (
     set overwrite_optimize=true
     set status=rest
@@ -152,7 +105,3 @@ if not exist "%~1" (
 )
 set status=
 exit /b
-
-:get_absolute_path
-set absolute_path=%~f1
-exit /b

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java

@@ -62,6 +62,9 @@ public class EmbulkCommandLine
                              "jruby_load_path",
                              Paths.get(oneJRubyLoadPathToAddLib).resolve("lib").toString());
             }
+            if (this.bundlePath == null) {
+                systemConfigJRubyLoadPath.put("jruby_use_default_embulk_gem_home", "true");
+            }
             return new EmbulkCommandLine(
                 this.arguments,
                 systemConfigJRubyLoadPath,

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java

@@ -693,7 +693,6 @@ public class EmbulkRun
 
     private void printGeneralUsage(final PrintStream out)
     {
-        final String gemHomeEnv = System.getenv("GEM_HOME");
         out.println("Embulk v" + this.embulkVersion);
         out.println("Usage: embulk [-vm-options] <command> [--options]");
         out.println("Commands:");
@@ -704,13 +703,15 @@ public class EmbulkRun
         out.println("   preview    <config.yml>                            # dry-run the bulk load without output and show preview.");
         out.println("   guess      <partial-config.yml> -o <output.yml>    # guess missing parameters to create a complete configuration file.");
         out.println("   gem        <install | list | help>                 # install a plugin or show installed plugins.");
-        out.println("                                                      # plugin path is " + (gemHomeEnv == null ? "(empty)" : gemHomeEnv));
         out.println("   new        <category> <name>                       # generates new plugin template");
         out.println("   migrate    <path>                                  # modify plugin code to use the latest Embulk plugin API");
         out.println("   example    [path]                                  # creates an example config file and csv file to try embulk.");
         out.println("   selfupdate [version]                               # upgrades embulk to the latest released version or to the specified version.");
         out.println("");
         out.println("VM options:");
+        out.println("   -E...                            Run an external script to configure environment variables in JVM");
+        out.println("                                    (Operations not just setting envs are not recommended nor guaranteed.");
+        out.println("                                     Expect side effects by running your external script at your own risk.)");
         out.println("   -J-O                             Disable JVM optimizations to speed up startup time (enabled by default if command is 'run')");
         out.println("   -J+O                             Enable JVM optimizations to speed up throughput");
         out.println("   -J...                            Set JVM options (use -J-help to see available options)");
@@ -730,12 +731,24 @@ public class EmbulkRun
         // Not |LocalContextScope.SINGLETON| to narrow down considerations.
         final ScriptingContainer localJRubyContainer =
             new ScriptingContainer(LocalContextScope.SINGLETHREAD, LocalVariableBehavior.PERSISTENT);
+
+        // NOTE: Same done in JRubyScriptingModule.
+        // Remember to update |org.embulk.jruby.JRubyScriptingModule| when these environment variables are changed.
+        localJRubyContainer.runScriptlet("ENV.delete('BUNDLE_GEMFILE')");
+        localJRubyContainer.runScriptlet("ENV['GEM_HOME'] = File.expand_path File.join(Java::java.lang.System.properties['user.home'], '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])");
+        localJRubyContainer.runScriptlet("ENV['GEM_PATH'] = ''");
+
         return localJRubyContainer;
     }
 
     private void callJRubyGem(final List<String> subcommandArguments)
     {
         final ScriptingContainer localJRubyContainer = createLocalJRubyScriptingContainer();
+
+        localJRubyContainer.runScriptlet("puts ''");
+        localJRubyContainer.runScriptlet("puts 'Gem plugin path is: %s' % (ENV.has_key?('GEM_HOME') ? ENV['GEM_HOME'] : '(empty)')");
+        localJRubyContainer.runScriptlet("puts ''");
+
         localJRubyContainer.runScriptlet("require 'rubygems/gem_runner'");
         localJRubyContainer.put("__internal_argv_java__", subcommandArguments);
         localJRubyContainer.runScriptlet("Gem::GemRunner.new.run Array.new(__internal_argv_java__)");

data/embulk-cli/src/main/sh/{selfrun.sh.template → selfrun.sh}

@@ -5,6 +5,12 @@ overwrite_optimize=""
 
 while true; do
     case "$1" in
+        -E*)
+            external_script="${1#-E}"
+            echo "Running an external shell script: $external_script"
+            . $external_script
+            shift
+            ;;
         "-J+O")
             overwrite_optimize="true"
             shift
@@ -44,50 +50,11 @@ while true; do
     esac
 done
 
-embulk_args="$@"
-
-while [ $# -gt 0 ] ; do
-    case "$1" in
-        "-b" | "--bundle")
-            shift
-            EMBULK_BUNDLE_PATH="$1"
-            export EMBULK_BUNDLE_PATH
-            shift
-            break
-            ;;
-        *)
-            shift
-            ;;
-    esac
-done
-
-if test -z ${EMBULK_BUNDLE_PATH}; then
-    unset EMBULK_BUNDLE_PATH
-    unset BUNDLE_GEMFILE
-    GEM_HOME="`cd && pwd`/.embulk/jruby/<%= ruby_version %>"
-    export GEM_HOME
-    GEM_PATH=""
-    export GEM_PATH
-else
-    if test ! -d ${EMBULK_BUNDLE_PATH}; then
-        echo "Directory not found: \"${EMBULK_BUNDLE_PATH}\""
-        exit 127
-    fi
-    BUNDLE_GEMFILE="`cd ${EMBULK_BUNDLE_PATH} && pwd`/Gemfile"
-    if test ! -f ${BUNDLE_GEMFILE}; then
-        echo "Gemfile not found: \"${BUNDLE_GEMFILE}\""
-        exit 127
-    fi
-    export BUNDLE_GEMFILE
-    unset GEM_HOME
-    unset GEM_PATH
-fi
-
 if test "$overwrite_optimize" = "true" -o "$default_optimize" -a "$overwrite_optimize" != "false"; then
     java_args="-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC $java_args"
 else
     java_args="-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none $java_args"
 fi
 
-exec java $java_args -jar "$0" $jruby_args $embulk_args
+exec java $java_args -jar "$0" $jruby_args "$@"
 exit 127

data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java

@@ -40,11 +40,7 @@ public class SelfrunTest {
             .replaceAll("java ",
                         "java -classpath "
                         + classpath.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\")
-                        + " org.embulk.cli.DummyMain ")
-            // The following version string emulates RbConfig::CONFIG["ruby_version"].
-            // RbConfig::CONFIG["ruby_version"] is different from org.jruby.runtime.Constants.RUBY_VERSION.
-            // https://github.com/jruby/jruby/blob/9.1.5.0/core/src/main/java/org/jruby/ext/rbconfig/RbConfigLibrary.java#L229-L235
-            .replace("<%= ruby_version %>", org.jruby.runtime.Constants.RUBY_MAJOR_VERSION + ".0");
+                        + " org.embulk.cli.DummyMain ");
 
         // Modify selfrun so that arguments are written in 'args.txt' .
         Files.write(fileSystem.getPath(testSelfrunFile.getAbsolutePath()),
@@ -292,9 +288,9 @@ public class SelfrunTest {
             folder = new File(folder, "embulk-cli");
         }
         if (System.getProperty("file.separator").equals("\\")) {
-            return new File(new File(new File(new File(folder, "src"), "main"), "bat"), "selfrun.bat.template");
+            return new File(new File(new File(new File(folder, "src"), "main"), "bat"), "selfrun.bat");
         } else {
-            return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh.template");
+            return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh");
         }
     }
 }

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -56,6 +56,7 @@ public class JRubyScriptingModule
         private final Injector injector;
         private final boolean useGlobalRubyRuntime;
         private final String gemHome;
+        private final boolean useDefaultEmbulkGemHome;
         private final List<String> jrubyClasspath;
         private final List<String> jrubyLoadPath;
         private final List<String> jrubyOptions;
@@ -71,6 +72,8 @@ public class JRubyScriptingModule
             this.useGlobalRubyRuntime = systemConfig.get(boolean.class, "use_global_ruby_runtime", false);
 
             this.gemHome = systemConfig.get(String.class, "gem_home", null);
+            this.useDefaultEmbulkGemHome =
+                systemConfig.get(String.class, "jruby_use_default_embulk_gem_home", "false").equals("true");
 
             // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
 
@@ -133,6 +136,13 @@ public class JRubyScriptingModule
         {
             LocalContextScope scope = (useGlobalRubyRuntime ? LocalContextScope.SINGLETON : LocalContextScope.SINGLETHREAD);
             ScriptingContainer jruby = new ScriptingContainer(scope, LocalVariableBehavior.PERSISTENT);
+            if (this.useDefaultEmbulkGemHome) {
+                // NOTE: Same done in "gem", "exec", and "irb" subcommands.
+                // Remember to update |org.embulk.cli.EmbulkRun| as well when these environment variables are changed.
+                jruby.runScriptlet("ENV.delete('BUNDLE_GEMFILE')");
+                jruby.runScriptlet("ENV['GEM_HOME'] = File.expand_path File.join(Java::java.lang.System.properties['user.home'], '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])");
+                jruby.runScriptlet("ENV['GEM_PATH'] = ''");
+            }
             final RubyInstanceConfig jrubyInstanceConfig = jruby.getProvider().getRubyInstanceConfig();
             for (final String jrubyOption : this.jrubyOptions) {
                 try {
@@ -250,14 +260,12 @@ public class JRubyScriptingModule
         private void setBundlerPluginSourceDirectory(final ScriptingContainer jruby, final String directory)
         {
             if (directory != null) {
-                /* Environment variables are set in the selfrun script or bin/embulk:
-                 *   ENV['EMBULK_BUNDLE_PATH']: set through '-b' | '--bundle', or inherit from the runtime environment
-                 *   ENV['BUNDLE_GEMFILE']: set for "ENV['EMBULK_BUNDLE_PATH']/Gemfile"
-                 *   ENV['GEM_HOME']: unset
-                 *   ENV['GEM_PATH']: unset
-                 */
+                jruby.put("__internal_bundler_plugin_source_directory__", directory);
+                jruby.runScriptlet("ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(__internal_bundler_plugin_source_directory__, 'Gemfile')");
 
                 // bundler is included in embulk-core.jar
+                jruby.runScriptlet("ENV.delete('GEM_HOME')");
+                jruby.runScriptlet("ENV.delete('GEM_PATH')");
                 jruby.runScriptlet("Gem.clear_paths");
                 jruby.runScriptlet("require 'bundler'");
 
@@ -270,13 +278,11 @@ public class JRubyScriptingModule
                 // It can cause insecure injections.
                 //
                 // add bundle directory path to load local plugins at ./embulk
-                jruby.put("__internal_bundler_plugin_source_directory__", directory);
                 jruby.runScriptlet("$LOAD_PATH << File.expand_path(__internal_bundler_plugin_source_directory__)");
                 jruby.remove("__internal_bundler_plugin_source_directory__");
             }
             else {
-                /* Environment variables are set in the selfrun script or bin/embulk:
-                 *   ENV['EMBULK_BUNDLE_PATH']: unset
+                /* Environment variables are set in ScriptingContainerProvider#get():
                  *   ENV['BUNDLE_GEMFILE']: unset
                  *   ENV['GEM_HOME']: set for "~/.embulk/jruby/${ruby-version}"
                  *   ENV['GEM_PATH']: set for ""

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.33
     release/release-0.8.32
     release/release-0.8.31
     release/release-0.8.30

data/embulk-docs/src/release/release-0.8.33.rst

@@ -0,0 +1,13 @@
+Release 0.8.33
+==================================
+
+General Changes
+------------------
+
+* Configure environment variables in JRuby again, instead of the selfrun scripts [#779]
+* Add a CLI option "-E" running an external script to configure environment variables [#778] [#790]
+
+
+Release Date
+------------------
+2017-09-15

data/lib/embulk/version.rb

@@ -3,7 +3,7 @@
 module Embulk
   @@warned = false
 
-  VERSION_INTERNAL = '0.8.32'
+  VERSION_INTERNAL = '0.8.33'
 
   DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
   def self.const_missing(name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.32
+  version: 0.8.33
 platform: java
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-07 00:00:00.000000000 Z
+date: 2017-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -155,10 +155,10 @@ files:
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang-2.4.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-cli-0.8.32.jar
-- classpath/embulk-core-0.8.32.jar
-- classpath/embulk-jruby-strptime-0.8.32.jar
-- classpath/embulk-standards-0.8.32.jar
+- classpath/embulk-cli-0.8.33.jar
+- classpath/embulk-core-0.8.33.jar
+- classpath/embulk-jruby-strptime-0.8.33.jar
+- classpath/embulk-standards-0.8.33.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -192,7 +192,7 @@ files:
 - classpath/validation-api-1.1.0.Final.jar
 - classpath/velocity-1.7.jar
 - embulk-cli/build.gradle
-- embulk-cli/src/main/bat/selfrun.bat.template
+- embulk-cli/src/main/bat/selfrun.bat
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java
 - embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLineException.java
@@ -213,7 +213,7 @@ files:
 - embulk-cli/src/main/java/org/embulk/cli/parse/HelpMessageLineDefinition.java
 - embulk-cli/src/main/java/org/embulk/cli/parse/OptionBehavior.java
 - embulk-cli/src/main/java/org/embulk/cli/parse/OptionDefinition.java
-- embulk-cli/src/main/sh/selfrun.sh.template
+- embulk-cli/src/main/sh/selfrun.sh
 - embulk-cli/src/test/java/org/embulk/cli/DummyMain.java
 - embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java
 - embulk-core/build.gradle
@@ -562,6 +562,7 @@ files:
 - embulk-docs/src/release/release-0.8.30.rst
 - embulk-docs/src/release/release-0.8.31.rst
 - embulk-docs/src/release/release-0.8.32.rst
+- embulk-docs/src/release/release-0.8.33.rst
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst
 - embulk-docs/src/release/release-0.8.6.rst