embulk 0.8.27-java → 0.8.28-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 61453ab9514646a642d390b971aeca9cb45ea066
-  data.tar.gz: 6c034da535e1687bcf3f6f9a75f12557b73d1f17
+  metadata.gz: fb3a3ad55003d5d87e050680472c891f8fef9de9
+  data.tar.gz: d9a164a0112947615e837dcf42a260eaaff716f0
 SHA512:
-  metadata.gz: 10c5d683990626cbe25dcfc6dcdc5f967d74d7420743f6c33ba1a1997765156b5e4ee1a0914737baa4f1ba80206864e404941b309e1b1efb9c05e2efae293c87
-  data.tar.gz: 9f21b019c2752622b0108b649859fbb1a39ea2420c2381e71df50ec683ee7f9a742fe0038096a5e6cf0e294fe5068ddf6f32ca8ee9e71bfd6f83cc01167b5027
+  metadata.gz: 9e6dfbcea73023ac911cc6260176a681d632e6a3023daa2cd9236504c91a30a9b00a121f633974b8ad7637a872da9be9c467b9bdc46182af06c7c09a4b715a0d
+  data.tar.gz: c4b28535cb4c54d2427f2b86a13f62e83c73dc89436b913ae8b6d2ac917dbc7e075f746c8eb6494a092bf615949b3989d39e6ec5a65342c3700d2cfc3e812b45

data/.travis.yml

@@ -8,7 +8,7 @@ sudo: false
 install:
   - pip install --user sphinx
 script:
-  - ./gradlew --info check rubyTest
+  - ./gradlew --info $GRADLE_TASK # each Travis process configures GRADLE_TASK by env.matrix
 after_success:
   - PATH="$HOME/.local/bin:$PATH" ./embulk-docs/push-gh-pages.sh
 env:
@@ -16,3 +16,13 @@ env:
     - GIT_USER_NAME=travis
     - GIT_USER_EMAIL=travis@embulk.org
     - secure: K5qT2PcCP/40dNW+1H4NZ6y1+GAZbyP/lMQ1tSMsAICGkMQ/A+Mp5wtnIGIsAf6JGcJ1PvpCoLE1V6wKFL5fEwxi4SRcTnZTh9PkeAk8dgezOMoX4EqeZiQAYv4MM2zKL+Gr6QivjmRA7I5jrZCo8JyaA5XfQ7ygjICKNJy8NaE=
+  # To speed up the Travis build, the build can be broken up into several parts by env.matrix.
+  # They'll run in parallel. Each Travis process set environment variables declared in each
+  # env.matrix like GRADLE_TASK. See: https://docs.travis-ci.com/user/speeding-up-the-build/
+  matrix:
+    - GRADLE_TASK=':embulk-core:check'
+    - GRADLE_TASK=':embulk-standards:check'
+    - GRADLE_TASK=':embulk-jruby-strptime:check'
+    - GRADLE_TASK=':embulk-test:check'
+    - GRADLE_TASK=':embulk-cli:check'
+    - GRADLE_TASK='rubyTest'

data/appveyor.yml

@@ -4,18 +4,18 @@ build:
   verbosity: detailed
 
 install:
- - ps: $zipPath = "$($env:USERPROFILE)\Handle.zip"
- - ps: (New-Object Net.WebClient).DownloadFile('https://download.sysinternals.com/files/Handle.zip', $zipPath)
- - ps: 7z x $zipPath -y -o"$env:APPVEYOR_BUILD_FOLDER" | Out-Null
+ - if not exist C:\Users\appveyor\handle mkdir C:\Users\appveyor\handle
+ - if not exist C:\Users\appveyor\handle\handle.exe appveyor DownloadFile "https://live.sysinternals.com/handle.exe" -FileName "C:\Users\appveyor\handle\handle.exe"
 
 # To avoid call-selfrun.bat FileNotFound exception, it installs and uses handle.exe as workaround provided by AppVeyor support.
 # see http://help.appveyor.com/discussions/problems/5975-the-process-cannot-access-the-file-because-it-is-being-used-by-another-process
 build_script:
-  - handle.exe -a -u C:\projects\embulk\embulk-cli\build\classes\test\org\embulk\cli\call-selfrun.bat -nobanner
+  - C:\Users\appveyor\handle\handle.exe -a -u C:\projects\embulk\embulk-cli\build\classes\test\org\embulk\cli\call-selfrun.bat -nobanner
   - gradlew.bat --info --no-daemon check rubyTest
 
 cache:
   - C:\Users\appveyor\.gradle
+  - C:\Users\appveyor\handle
 
 environment:
   matrix:

data/bin/embulk

@@ -1,22 +1,33 @@
 #!/usr/bin/env ruby
 
 if RUBY_PLATFORM =~ /java/i
+  STDERR.puts "################################################################################"
+  STDERR.puts "[WARN] embulk.gem is deprecated, and will be removed from v0.9."
+  STDERR.puts "[WARN] embulk.gem's bin/embulk will be also removed from v0.9."
+  STDERR.puts "[WARN] Use the jar version installed from http://dl.embulk.org/ instead."
+  STDERR.puts "[WARN] See the issue and comment at: https://github.com/embulk/embulk/issues/628"
+  STDERR.puts "################################################################################"
+  STDERR.puts ""
+
+  require 'java'
+
   if ENV['EMBULK_BIN_ENABLE_BUNDLE'] == File.expand_path(__FILE__)
-    # bin/embulk is run by CRuby (embulk gem for CRuby is installed). enable embulk_bundle.
+    # bin/embulk is started by CRuby (embulk gem for CRuby is installed), and re-run by JRuby at the end of the script.
+    # Bundler may be enabled in this path.
     ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
-    # include -cp CLASSPATH to LOAD_PATH so that embulk_bundle.rb can load bundler included in embulk-core.jar
-    $LOAD_PATH << "uri:classloader:/"
 
-    # Handle environment variables before entering embulk_bundle.rb.
+    # Handle environment variables before entering org.embulk.cli.EmbulkBundle.
     # See: https://github.com/embulk/embulk/pull/604
 
     bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
     bundle_path = nil if bundle_path.empty?
 
     # Search for -b or --bundle, and remove it.
-    if ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
-      ARGV.slice!(bundle_path_index, 2)[1]
+    bundle_option_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+    if bundle_option_index
+      ARGV.slice!(bundle_option_index, 2)[1]
     end
+
     if bundle_path
       ENV['EMBULK_BUNDLE_PATH'] = bundle_path
       ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
@@ -33,11 +44,24 @@ if RUBY_PLATFORM =~ /java/i
       ENV.delete('BUNDLE_GEMFILE')
     end
 
-    require_relative '../lib/embulk/command/embulk_bundle'
+    require 'embulk/version'
+    Java::org.embulk.cli.EmbulkBundle::checkBundleWithEmbulkVersion(ARGV,
+                                                                    Java::JavaUtil::ArrayList.new,
+                                                                    Embulk::VERSION_INTERNAL)
   else
-    # bin/embulk is run by JRuby (embulk gem for JRuby is installed). disable embulk_bundle not to bother the JRuby's bundler
+    # bin/embulk is started by JRuby (embulk gem for JRuby is installed).
+    # Bundler must be disabled in this path not to bother the JRuby's bundler.
     $LOAD_PATH << File.expand_path('../lib', File.dirname(__FILE__))
-    require 'embulk/command/embulk_main'
+    Dir.glob("#{File.expand_path('../classpath', File.dirname(__FILE__))}/*").each do |f_jar|
+      $CLASSPATH << f_jar
+    end
+    require 'embulk/version'
+    # The ScriptingContainer instance below is the same JRuby instance with the one running this bin/embulk itself
+    # as it's LocalContextScope.SINGLETON. The instance then inherits $LOAD_PATH and $CLASSPATH configured above.
+    global_jruby_container = Java::org.jruby.embed.ScriptingContainer.new(
+      Java::org.jruby.embed.LocalContextScope::SINGLETON, Java::org.jruby.embed.LocalVariableBehavior::PERSISTENT)
+    embulk_run = Java::org.embulk.cli.EmbulkRun.new(Embulk::VERSION_INTERNAL, global_jruby_container)
+    embulk_run.run(Java::JavaUtil::ArrayList.new(ARGV), Java::JavaUtil::ArrayList.new)
   end
   exit 0
 end

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.27'
+    version = '0.8.28'
 
     ext {
         jrubyVersion = '9.1.5.0'
@@ -209,18 +209,27 @@ task classpath(dependsOn: ['build', ':embulk-cli:classpath']) { doLast {} }
 
 clean { delete 'classpath' }
 
+//
+// Gets RbConfig::CONFIG['ruby_version'] from the same version of JRuby with Embulk's embedded.
+//
+task retrieveRubyVersion(type: JRubyExec) {
+    standardOutput = new ByteArrayOutputStream()
+    jrubyArgs '-e', "print RbConfig::CONFIG['ruby_version']"
+}
+
 //
 // cli task
 //
-task cli(dependsOn: ':embulk-cli:shadowJar') {
+task cli(dependsOn: ['retrieveRubyVersion', ':embulk-cli:shadowJar']) {
     doLast {
+        def rubyVersion = retrieveRubyVersion.standardOutput.toString().trim()
         file('pkg').mkdirs()
         File f = file("pkg/embulk-${project.version}.jar")
         f.write("")
         f.append("\n: <<END_OF_EMBULK_SELFRUN_BATCH_PART\r\n")
-        f.append(file("embulk-cli/src/main/bat/selfrun.bat").readBytes())
+        f.append(file("embulk-cli/src/main/bat/selfrun.bat.template").getText().replace("<%= ruby_version %>", rubyVersion))
         f.append("\r\nEND_OF_EMBULK_SELFRUN_BATCH_PART\r\n\n")
-        f.append(file("embulk-cli/src/main/sh/selfrun.sh").readBytes())
+        f.append(file("embulk-cli/src/main/sh/selfrun.sh.template").getText().replace("<%= ruby_version %>", rubyVersion))
         f.append(file("embulk-cli/build/libs/embulk-cli-${project.version}-all.jar").readBytes())
         f.setExecutable(true)
     }
@@ -243,6 +252,10 @@ project(':embulk-cli') {
         // - embulk-core: MavenPluginSource and the related (from v0.8.26)
         relocate 'org.eclipse.aether', 'org.embulk.third_party.org.eclipse.aether'
 
+        // 'org.apache.commons.cli' is used in:
+        // - embulk-cli: EmbulkCommandLineParser (from v0.8.28)
+        relocate 'org.apache.commons.cli', 'org.embulk.third_party.org.apache.commons.cli'
+
         // NOTE: This 'Implementation-Version' in the manifest is referred to provide the Embulk version at runtime.
         // See also: embulk-core/src/main/java/org/embulk/EmbulkVersion.java
         manifest {

data/embulk-cli/build.gradle

@@ -1,6 +1,7 @@
 dependencies {
     compile project(':embulk-core')
     compile project(':embulk-standards')
+    compile 'commons-cli:commons-cli:1.3.1'
     compile 'org.apache.maven:maven-artifact:3.3.9'
     compile 'org.apache.velocity:velocity:1.7'
 

data/embulk-cli/src/main/bat/{selfrun.bat → selfrun.bat.template}

@@ -79,8 +79,7 @@ endlocal && set BUNDLE_GEMFILE=%bundle_gemfile%
 setlocal enabledelayedexpansion
 
 if not defined EMBULK_BUNDLE_PATH (
-    for /f "delims=" %%w in ('java -cp %this% org.jruby.Main -e "print RbConfig::CONFIG['ruby_version']"') do set ruby_version=%%w
-    set gem_home=%USERPROFILE%\.embulk\jruby\!ruby_version!
+    set gem_home=%USERPROFILE%\.embulk\jruby\<%= ruby_version %>
 ) else (
     set gem_home=
 )

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkArguments.java

@@ -0,0 +1,54 @@
+package org.embulk.cli;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class EmbulkArguments
+{
+    private EmbulkArguments(final EmbulkSubcommand subcommand, final List<String> subcommandArguments)
+    {
+        this.subcommand = subcommand;
+        this.subcommandArguments = Collections.unmodifiableList(subcommandArguments);
+    }
+
+    public static EmbulkArguments extract(final List<String> arguments)
+            throws EmbulkCommandLineException
+    {
+        final ArrayList<String> subcommandArguments = new ArrayList<String>();
+
+        EmbulkSubcommand subcommand = null;
+        for (final String argument : arguments) {
+            if (subcommand == null && (!argument.startsWith("-"))) {
+                subcommand = EmbulkSubcommand.of(argument);
+            }
+            else if (subcommand == null && (argument.equals("-b") || argument.equals("--bundle"))) {
+                throw new EmbulkCommandLineException("\"-b\" or \"--bundle\" before a subcommand is not supported.");
+            }
+            else if (argument.equals("-version")) {
+                return new EmbulkArguments(EmbulkSubcommand.VERSION_ERR, new ArrayList<String>());
+            }
+            else if (argument.equals("--version")) {
+                return new EmbulkArguments(EmbulkSubcommand.VERSION_OUT, new ArrayList<String>());
+            }
+            else {
+                subcommandArguments.add(argument);
+            }
+        }
+
+        return new EmbulkArguments(subcommand, subcommandArguments);
+    }
+
+    public final EmbulkSubcommand getSubcommand()
+    {
+        return this.subcommand;
+    }
+
+    public final List<String> getSubcommandArguments()
+    {
+        return this.subcommandArguments;
+    }
+
+    private final EmbulkSubcommand subcommand;
+    private final List<String> subcommandArguments;
+}

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkBundle.java

@@ -0,0 +1,44 @@
+package org.embulk.cli;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import org.embulk.EmbulkVersion;
+import org.jruby.embed.ScriptingContainer;
+
+public class EmbulkBundle
+{
+    public static void checkBundle(
+            final String[] embulkArgs,
+            final List<String> jrubyOptions)
+    {
+        checkBundleWithEmbulkVersion(embulkArgs, jrubyOptions, EmbulkVersion.VERSION);
+    }
+
+    // It accepts |embulkVersion| so that it can receive Embulk's version from Ruby (bin/embulk).
+    public static void checkBundleWithEmbulkVersion(
+            final String[] embulkArgs,
+            final List<String> jrubyOptions,
+            final String embulkVersion)
+    {
+        final String bundlePath = System.getenv("EMBULK_BUNDLE_PATH");
+
+        final ScriptingContainer globalJRubyContainer =
+            EmbulkGlobalJRubyScriptingContainer.setup(embulkArgs, jrubyOptions, bundlePath, System.err);
+
+        // NOTE: It was written in Ruby as follows in case |bundlePath| != null:
+        //   begin
+        //     require 'embulk/command/embulk_main'
+        //   rescue LoadError
+        //     $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+        //     require 'embulk/command/embulk_main'
+        //   end
+        //
+        // NOTE: It was written in Ruby as follows in case |bundlePath| == null::
+        //   require 'embulk/command/embulk_main'
+        //
+        // TODO: Consider handling LoadError or similar errors.
+        final EmbulkRun runner = new EmbulkRun(embulkVersion, globalJRubyContainer);
+        runner.run(Collections.unmodifiableList(Arrays.asList(embulkArgs)), jrubyOptions);
+    }
+}

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java

@@ -0,0 +1,256 @@
+package org.embulk.cli;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class EmbulkCommandLine
+{
+    private EmbulkCommandLine(
+            final List<String> arguments,
+            final Map<String, Object> systemConfig,
+            final String bundle,
+            final String bundlePath,
+            final List<String> classpath,
+            final String configDiff,
+            final boolean force,
+            final String format,
+            final List<String> load,
+            final List<String> loadPath,
+            final String output,
+            final String resumeState)
+    {
+        this.arguments = Collections.unmodifiableList(arguments);
+        this.systemConfig = Collections.unmodifiableMap(systemConfig);
+        this.bundle = bundle;
+        this.bundlePath = bundlePath;
+        this.classpath = Collections.unmodifiableList(classpath);
+        this.configDiff = configDiff;
+        this.force = force;
+        this.format = format;
+        this.load = Collections.unmodifiableList(load);
+        this.loadPath = Collections.unmodifiableList(loadPath);
+        this.output = output;
+        this.resumeState = resumeState;
+    }
+
+    public static final class Builder
+    {
+        public Builder()
+        {
+            this.arguments = new ArrayList<String>();
+            this.systemConfig = new HashMap<String, Object>();
+            this.bundle = null;
+            this.bundlePath = null;
+            this.classpath = new ArrayList<String>();
+            this.configDiff = null;
+            this.force = false;
+            this.format = null;
+            this.load = new ArrayList<String>();
+            this.loadPath = new ArrayList<String>();
+            this.output = null;
+            this.resumeState = null;
+        }
+
+        public EmbulkCommandLine build()
+        {
+            return new EmbulkCommandLine(
+                this.arguments,
+                this.systemConfig,
+                this.bundle,
+                this.bundlePath,
+                this.classpath,
+                this.configDiff,
+                this.force,
+                this.format,
+                this.load,
+                this.loadPath,
+                this.output,
+                this.resumeState);
+        }
+
+        public Builder addArguments(final List<String> arguments)
+        {
+            this.arguments.addAll(arguments);
+            return this;
+        }
+
+        public Builder setSystemConfig(final String key, final String value)
+        {
+            this.systemConfig.put(key, value);
+            return this;
+        }
+
+        public Builder addSystemConfig(final String key, final String value)
+        {
+            final Object existingValue = this.systemConfig.get(key);
+            if (existingValue != null && existingValue instanceof String) {
+                this.systemConfig.put(key, Arrays.asList((String) existingValue, value));
+            }
+            else if (existingValue != null && existingValue instanceof List) {
+                @SuppressWarnings("unchecked")
+                final ArrayList<String> newList = new ArrayList<String>((List<String>) existingValue);
+                newList.add(value);
+                this.systemConfig.put(key, Collections.unmodifiableList(newList));
+            }
+            else {
+                this.systemConfig.put(key, Arrays.asList(value));
+            }
+            return this;
+        }
+
+        public Builder setBundle(final String bundle)
+        {
+            this.bundle = bundle;
+            return this;
+        }
+
+        public Builder setBundlePath(final String bundlePath)
+        {
+            this.bundlePath = bundlePath;
+            return this;
+        }
+
+        public Builder addClasspath(final String classpath)
+        {
+            this.classpath.add(classpath);
+            return this;
+        }
+
+        public Builder setConfigDiff(final String configDiff)
+        {
+            this.configDiff = configDiff;
+            return this;
+        }
+
+        public Builder setForce(final boolean force)
+        {
+            this.force = force;
+            return this;
+        }
+
+        public Builder setFormat(final String format)
+        {
+            this.format = format;
+            return this;
+        }
+
+        public Builder addLoad(final String load)
+        {
+            this.load.add(load);
+            return this;
+        }
+
+        public Builder addLoadPath(final String loadPath)
+        {
+            this.loadPath.add(loadPath);
+            return this;
+        }
+
+        public Builder setOutput(final String output)
+        {
+            this.output = output;
+            return this;
+        }
+
+        public Builder setResumeState(final String resumeState)
+        {
+            this.resumeState = resumeState;
+            return this;
+        }
+
+        private ArrayList<String> arguments;
+        private HashMap<String, Object> systemConfig;
+        private String bundle;
+        private String bundlePath;
+        private ArrayList<String> classpath;
+        private String configDiff;
+        private boolean force;
+        private String format;
+        private ArrayList<String> load;
+        private ArrayList<String> loadPath;
+        private String output;
+        private String resumeState;
+    }
+
+    public static Builder builder()
+    {
+        return new Builder();
+    }
+
+    public final List<String> getArguments()
+    {
+        return this.arguments;
+    }
+
+    public final Map<String, Object> getSystemConfig()
+    {
+        return this.systemConfig;
+    }
+
+    public final String getBundle()
+    {
+        return this.bundle;
+    }
+
+    public final String getBundlePath()
+    {
+        return this.bundlePath;
+    }
+
+    public final List<String> getClasspath()
+    {
+        return this.classpath;
+    }
+
+    public final String getConfigDiff()
+    {
+        return this.configDiff;
+    }
+
+    public final boolean getForce()
+    {
+        return this.force;
+    }
+
+    public final String getFormat()
+    {
+        return this.format;
+    }
+
+    public final List<String> getLoad()
+    {
+        return this.load;
+    }
+
+    public final List<String> getLoadPath()
+    {
+        return this.loadPath;
+    }
+
+    public final String getOutput()
+    {
+        return this.output;
+    }
+
+    public final String getResumeState()
+    {
+        return this.resumeState;
+    }
+
+    private final List<String> arguments;
+    private final Map<String, Object> systemConfig;
+    private final String bundle;
+    private final String bundlePath;
+    private final List<String> classpath;
+    private final String configDiff;
+    private final boolean force;
+    private final String format;
+    private final List<String> load;
+    private final List<String> loadPath;
+    private final String output;
+    private final String resumeState;
+}