embulk 0.8.31-java → 0.8.32-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b75f4f5fd6e21885fe49a8af1468a43ee0989785
-  data.tar.gz: 2c12d170b838e916a67e6e79ae9d85b985c77f4a
+  metadata.gz: 2df25ebf0f892234350ee64d8ed0b2d0d09fc1a2
+  data.tar.gz: e2b776f7de4f56904dce422fb3422ff221cca124
 SHA512:
-  metadata.gz: 1aafd34ed37453dd11673facebc87ed6e32127ab93bbcde55fe16e8469c51ac968098d058616ad78a536227b4c4fb1eb1598c3c85c4d72e31133fb0f84449453
-  data.tar.gz: 0e4b5f0b63b1dd34d357b1fa6ea585a4be3ee74c6e9ed428d383de8100b036a789a326bd3b626306220c11b7fb9292c0cfae56d11e14626cb34f493a53e5e78c
+  metadata.gz: 927d56589043664d0064a0d4c4996735325daaeef47de6979f15076961c8eaf9a0acb3a28c5ccd6ecbc635ca88acb32028b57cb531b6728dc594bd7d3318655e
+  data.tar.gz: 47adef5189e27a8810aec2c9d4bd4c6228d55391e9d300ca1cdb3b4233e1e7b180c68acc9a4dc5d500137006a7ad9d991ea57062161c05783d5402f393641e9d

data/build.gradle

@@ -1,3 +1,11 @@
+// TODO: Remove this block once rubygems.lasagna.io is back, or jruby-gradle-jar-plugin is upgraded to 1.5.0.
+// See also: https://github.com/jruby-gradle/jruby-gradle-plugin/issues/297
+// jruby-gradle-jar-plugin is not upgraded yet because its 1.5.0 depends on Java 8.
+repositories {
+    mavenLocal()
+    maven { url 'http://rubygems-proxy.torquebox.org/releases' }
+}
+
 buildscript {
     repositories { jcenter() }
     dependencies {
@@ -16,7 +24,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.31'
+    version = '0.8.32'
 
     ext {
         jrubyVersion = '9.1.5.0'

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkCommandLine.java

@@ -1,5 +1,6 @@
 package org.embulk.cli;
 
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -12,27 +13,19 @@ public class EmbulkCommandLine
     private EmbulkCommandLine(
             final List<String> arguments,
             final Map<String, Object> systemConfig,
-            final String bundle,
             final String bundlePath,
-            final List<String> classpath,
             final String configDiff,
             final boolean force,
             final String format,
-            final List<String> load,
-            final List<String> loadPath,
             final String output,
             final String resumeState)
     {
         this.arguments = Collections.unmodifiableList(arguments);
         this.systemConfig = Collections.unmodifiableMap(systemConfig);
-        this.bundle = bundle;
         this.bundlePath = bundlePath;
-        this.classpath = Collections.unmodifiableList(classpath);
         this.configDiff = configDiff;
         this.force = force;
         this.format = format;
-        this.load = Collections.unmodifiableList(load);
-        this.loadPath = Collections.unmodifiableList(loadPath);
         this.output = output;
         this.resumeState = resumeState;
     }
@@ -43,9 +36,7 @@ public class EmbulkCommandLine
         {
             this.arguments = new ArrayList<String>();
             this.systemConfig = new HashMap<String, Object>();
-            this.bundle = null;
             this.bundlePath = null;
-            this.classpath = new ArrayList<String>();
             this.configDiff = null;
             this.force = false;
             this.format = null;
@@ -57,17 +48,27 @@ public class EmbulkCommandLine
 
         public EmbulkCommandLine build()
         {
+            final HashMap<String, Object> systemConfigJRubyLoadPath = new HashMap<String, Object>(this.systemConfig);
+            // first $LOAD_PATH has highet priority. later load_paths should have highest priority.
+            for (final String oneJRubyLoadPath : this.loadPath) {  // "-L"
+                addInHashMap(systemConfigJRubyLoadPath, "jruby_load_path", oneJRubyLoadPath);
+            }
+            // Gem::StubSpecification is an internal API that seems chainging often.
+            // Gem::Specification.add_spec is deprecated also. Therefore, here makes
+            // -L <path> option alias of -I <path>/lib by assuming that *.gemspec file
+            // always has require_paths = ["lib"].
+            for (final String oneJRubyLoadPathToAddLib : this.load) {  // "-I"
+                addInHashMap(systemConfigJRubyLoadPath,
+                             "jruby_load_path",
+                             Paths.get(oneJRubyLoadPathToAddLib).resolve("lib").toString());
+            }
             return new EmbulkCommandLine(
                 this.arguments,
-                this.systemConfig,
-                this.bundle,
+                systemConfigJRubyLoadPath,
                 this.bundlePath,
-                this.classpath,
                 this.configDiff,
                 this.force,
                 this.format,
-                this.load,
-                this.loadPath,
                 this.output,
                 this.resumeState);
         }
@@ -86,25 +87,7 @@ public class EmbulkCommandLine
 
         public Builder addSystemConfig(final String key, final String value)
         {
-            final Object existingValue = this.systemConfig.get(key);
-            if (existingValue != null && existingValue instanceof String) {
-                this.systemConfig.put(key, Arrays.asList((String) existingValue, value));
-            }
-            else if (existingValue != null && existingValue instanceof List) {
-                @SuppressWarnings("unchecked")
-                final ArrayList<String> newList = new ArrayList<String>((List<String>) existingValue);
-                newList.add(value);
-                this.systemConfig.put(key, Collections.unmodifiableList(newList));
-            }
-            else {
-                this.systemConfig.put(key, Arrays.asList(value));
-            }
-            return this;
-        }
-
-        public Builder setBundle(final String bundle)
-        {
-            this.bundle = bundle;
+            addInHashMap(this.systemConfig, key, value);
             return this;
         }
 
@@ -114,12 +97,6 @@ public class EmbulkCommandLine
             return this;
         }
 
-        public Builder addClasspath(final String classpath)
-        {
-            this.classpath.add(classpath);
-            return this;
-        }
-
         public Builder setConfigDiff(final String configDiff)
         {
             this.configDiff = configDiff;
@@ -162,11 +139,26 @@ public class EmbulkCommandLine
             return this;
         }
 
+        private static void addInHashMap(final HashMap<String, Object> map, final String key, final String value)
+        {
+            final Object existingValue = map.get(key);
+            if (existingValue != null && existingValue instanceof String) {
+                map.put(key, Arrays.asList((String) existingValue, value));
+            }
+            else if (existingValue != null && existingValue instanceof List) {
+                @SuppressWarnings("unchecked")
+                final ArrayList<String> newList = new ArrayList<String>((List<String>) existingValue);
+                newList.add(value);
+                map.put(key, Collections.unmodifiableList(newList));
+            }
+            else {
+                map.put(key, Arrays.asList(value));
+            }
+        }
+
         private ArrayList<String> arguments;
         private HashMap<String, Object> systemConfig;
-        private String bundle;
         private String bundlePath;
-        private ArrayList<String> classpath;
         private String configDiff;
         private boolean force;
         private String format;
@@ -191,21 +183,11 @@ public class EmbulkCommandLine
         return this.systemConfig;
     }
 
-    public final String getBundle()
-    {
-        return this.bundle;
-    }
-
     public final String getBundlePath()
     {
         return this.bundlePath;
     }
 
-    public final List<String> getClasspath()
-    {
-        return this.classpath;
-    }
-
     public final String getConfigDiff()
     {
         return this.configDiff;
@@ -221,16 +203,6 @@ public class EmbulkCommandLine
         return this.format;
     }
 
-    public final List<String> getLoad()
-    {
-        return this.load;
-    }
-
-    public final List<String> getLoadPath()
-    {
-        return this.loadPath;
-    }
-
     public final String getOutput()
     {
         return this.output;
@@ -243,14 +215,10 @@ public class EmbulkCommandLine
 
     private final List<String> arguments;
     private final Map<String, Object> systemConfig;
-    private final String bundle;
     private final String bundlePath;
-    private final List<String> classpath;
     private final String configDiff;
     private final boolean force;
     private final String format;
-    private final List<String> load;
-    private final List<String> loadPath;
     private final String output;
     private final String resumeState;
 }

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkMigrate.java

@@ -13,6 +13,7 @@ import java.nio.file.SimpleFileVisitor;
 import java.nio.file.StandardCopyOption;
 import java.nio.file.StandardOpenOption;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.nio.file.attribute.PosixFilePermission;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -94,7 +95,9 @@ public class EmbulkMigrate
 
         // upgrade gradle version
         if (migrator.match("gradle/wrapper/gradle-wrapper.properties", GRADLE_VERSION_IN_WRAPPER)) {
-            // gradle < 3.2.1
+            // gradle < 4.1
+            migrator.copy("embulk/data/new/java/gradlew", "gradlew");
+            migrator.setExecutable("gradlew");
             migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.properties",
                           "gradle/wrapper/gradle-wrapper.properties");
             migrator.copy("embulk/data/new/java/gradle/wrapper/gradle-wrapper.jar",
@@ -410,6 +413,18 @@ public class EmbulkMigrate
             }
         }
 
+        private void setExecutable(String targetFileName)
+                throws IOException
+        {
+            final Path targetPath = this.basePath.resolve(targetFileName);
+            final Set<PosixFilePermission> permissions =
+                    new HashSet<PosixFilePermission>(Files.getPosixFilePermissions(targetPath));
+            permissions.add(PosixFilePermission.OWNER_EXECUTE);
+            permissions.add(PosixFilePermission.GROUP_EXECUTE);
+            permissions.add(PosixFilePermission.OTHERS_EXECUTE);
+            Files.setPosixFilePermissions(targetPath, permissions);
+        }
+
         private final Path basePath;
         private final Set<Path> modifiedFiles;
     }

data/embulk-cli/src/main/java/org/embulk/cli/EmbulkRun.java

@@ -84,7 +84,7 @@ public class EmbulkRun
             final EmbulkCommandLine commandLine;
             try {
                 commandLine = parser.parse(
-                    subcommandArguments, new PrintWriter(System.out), new PrintWriter(System.err));
+                    subcommandArguments, jrubyOptions, new PrintWriter(System.out), new PrintWriter(System.err));
             }
             catch (EmbulkCommandLineParseException ex) {
                 parser.printHelp(System.err);
@@ -446,13 +446,7 @@ public class EmbulkRun
             // embulk classes
             // NOTE: |EmbulkSetup.setup| returns |EmbulkEmbed| while it stores Ruby |Embulk::EmbulkRunner(EmbulkEmbed)|
             // into Ruby |Embulk::Runner|.
-            final EmbulkRunner runner = EmbulkSetup.setup(
-                jrubyOptions,
-                commandLine.getSystemConfig(),
-                commandLine.getLoadPath(),
-                commandLine.getLoad(),
-                commandLine.getClasspath(),
-                commandLine.getBundle());
+            final EmbulkRunner runner = EmbulkSetup.setup(commandLine.getSystemConfig());
 
             final Path configDiffPath =
                 (commandLine.getConfigDiff() == null ? null : Paths.get(commandLine.getConfigDiff()));
@@ -628,7 +622,7 @@ public class EmbulkRun
                 {
                     final String[] classpaths = argument.split("\\" + java.io.File.pathSeparator);
                     for (final String classpath : classpaths) {
-                        commandLineBuilder.addClasspath(classpath);
+                        commandLineBuilder.addSystemConfig("jruby_classpath", classpath);
                     }
                 }
             }));
@@ -641,7 +635,7 @@ public class EmbulkRun
             {
                 public void behave(final EmbulkCommandLine.Builder commandLineBuilder, final String argument)
                 {
-                    commandLineBuilder.setBundle(argument);
+                    commandLineBuilder.setSystemConfig("jruby_global_bundler_plugin_source_directory", argument);
                 }
             }));
     }

data/embulk-cli/src/main/java/org/embulk/cli/parse/EmbulkCommandLineParser.java

@@ -113,6 +113,7 @@ public class EmbulkCommandLineParser
     }
 
     public EmbulkCommandLine parse(final List<String> argsEmbulk,
+                                   final List<String> jrubyOptions,
                                    final PrintWriter helpPrintWriter,
                                    final PrintWriter errorPrintWriter)
             throws EmbulkCommandLineParseException, EmbulkCommandLineHelpRequired
@@ -144,6 +145,9 @@ public class EmbulkCommandLineParser
         }
 
         final EmbulkCommandLine.Builder commandLineBuilder = EmbulkCommandLine.builder();
+        for (final String jrubyOption : jrubyOptions) {
+            commandLineBuilder.addSystemConfig("jruby_command_line_options", jrubyOption);
+        }
         commandLineBuilder.addArguments(arguments);
         for (final Option cliOptionSpecified : cliCommandLine.getOptions()) {
             final OptionDefinition optionDefinitionSpecified =

data/embulk-core/build.gradle

@@ -1,3 +1,11 @@
+// TODO: Remove this block once rubygems.lasagna.io is back, or jruby-gradle-jar-plugin is upgraded to 1.5.0.
+// See also: https://github.com/jruby-gradle/jruby-gradle-plugin/issues/297
+// jruby-gradle-jar-plugin is not upgraded yet because its 1.5.0 depends on Java 8.
+repositories {
+    mavenLocal()
+    maven { url 'http://rubygems-proxy.torquebox.org/releases' }
+}
+
 apply plugin: "com.github.jruby-gradle.jar"
 
 // include ruby scripts to jar. don't use sourceSets.main.resources.srcDirs

data/embulk-core/src/main/java/org/embulk/EmbulkRunner.java

@@ -39,19 +39,9 @@ public class EmbulkRunner
 {
     // |EmbulkSetup.setup| initializes:
     // new EmbulkRunner(embed)
-    public EmbulkRunner(final EmbulkEmbed embed,
-                        final List<String> jrubyOptions,
-                        final List<String> loadPaths,
-                        final List<String> pluginPaths,
-                        final List<String> classpaths,
-                        final String bundlePath)
+    public EmbulkRunner(final EmbulkEmbed embed)
     {
         this.embed = embed;  // org.embulk.EmbulkEmbed
-        this.jrubyOptions = jrubyOptions;
-        this.loadPaths = loadPaths;
-        this.pluginPaths = pluginPaths;
-        this.classpaths = classpaths;
-        this.bundlePath = bundlePath;
     }
 
     /**
@@ -228,8 +218,7 @@ public class EmbulkRunner
     private void guessInternal(final ConfigSource configSource, final Path outputPath)
             throws IOException
     {
-        initializeGlobalJRubyScriptingContainer(
-            this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
+        initializeGlobalJRubyScriptingContainer();
 
         try {
             checkFileWritable(outputPath);
@@ -253,8 +242,7 @@ public class EmbulkRunner
     private void previewInternal(final ConfigSource configSource, final String format)
             throws IOException
     {
-        initializeGlobalJRubyScriptingContainer(
-            this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
+        initializeGlobalJRubyScriptingContainer();
 
         final PreviewResult previewResult = this.embed.preview(configSource);
         final ModelManager modelManager = this.embed.getModelManager();
@@ -283,8 +271,7 @@ public class EmbulkRunner
             final Path resumeStatePath)
             throws IOException
     {
-        initializeGlobalJRubyScriptingContainer(
-            this.jrubyOptions, this.loadPaths, this.pluginPaths, this.classpaths, this.bundlePath);
+        initializeGlobalJRubyScriptingContainer();
 
         try {
             checkFileWritable(outputPath);
@@ -541,46 +528,48 @@ public class EmbulkRunner
     // end
 
     // TODO: Check if it is required to process JRuby options.
-    private void initializeGlobalJRubyScriptingContainer(
-            final List<String> jrubyOptions,
-            final List<String> loadPaths,
-            final List<String> pluginPaths,
-            final List<String> classpaths,
-            final String bundlePath)
+    private void initializeGlobalJRubyScriptingContainer()
     {
         final ScriptingContainer globalJRubyContainer =
-            EmbulkGlobalJRubyScriptingContainer.setup(jrubyOptions, bundlePath, System.err);
-
-        // first $LOAD_PATH has highet priority. later load_paths should have highest priority.
-        for (final String loadPath : loadPaths) {
-            // ruby script directory (use unshift to make it highest priority)
-            globalJRubyContainer.put("__internal_load_path__", loadPath);
-            globalJRubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(__internal_load_path__)");
-            globalJRubyContainer.remove("__internal_load_path__");
-        }
-
-        // # Gem::StubSpecification is an internal API that seems chainging often.
-        // # Gem::Specification.add_spec is deprecated also. Therefore, here makes
-        // # -L <path> option alias of -I <path>/lib by assuming that *.gemspec file
-        // # always has require_paths = ["lib"].
-        for (final String pluginPath : pluginPaths) {
-            globalJRubyContainer.put("__internal_plugin_path__", pluginPath);
-            globalJRubyContainer.runScriptlet("$LOAD_PATH.unshift File.expand_path(File.join(__internal_plugin_path__, 'lib'))");
-            globalJRubyContainer.remove("__internal_plugin_path__");
-        }
-
-        for (final String classpath : classpaths) {
-            globalJRubyContainer.put("__internal_classpath__", classpath);
-            // $CLASSPATH object doesn't have concat method
-            globalJRubyContainer.runScriptlet("$CLASSPATH << __internal_classpath__");
-            globalJRubyContainer.remove("__internal_classpath__");
-        }
-
-        // see also embulk/java/bootstrap.rb loaded by JRubyScriptingModule
-        globalJRubyContainer.runScriptlet("module Embulk; end");
-        globalJRubyContainer.put("__internal_embulk_runner_java__", this);
-        globalJRubyContainer.runScriptlet("Embulk.const_set :Runner, Embulk::EmbulkRunner.new(__internal_embulk_runner_java__)");
-        globalJRubyContainer.remove("__internal_embulk_runner_java__");
+            new ScriptingContainer(LocalContextScope.SINGLETON, LocalVariableBehavior.PERSISTENT);
+
+        // TODO: Remove the Embulk::Runner definition after confirming nobody uses Embulk::Runner from Java.
+        globalJRubyContainer.put("__internal_runner_java__", this);
+        globalJRubyContainer.runScriptlet(
+            "class DummyEmbulkRunner\n" +
+            "  def initialize(runner_orig)\n" +
+            "    @runner_orig = runner_orig\n" +
+            "  end\n" +
+            "  def guess(config, options={})\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
+            "    STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts ''\n" +
+            "    @runner_orig.guess(config, options)\n" +
+            "  end\n" +
+            "  def preview(config, options={})\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
+            "    STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts ''\n" +
+            "    @runner_orig.preview(config, options)\n" +
+            "  end\n" +
+            "  def run(config, options={})\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts '[WARN] Embulk::Runner will be no longer defined when Embulk runs from Java.'\n" +
+            "    STDERR.puts '[WARN] Comment at https://github.com/embulk/embulk/issues/766 if you see this.'\n" +
+            "    STDERR.puts '################################################################################'\n" +
+            "    STDERR.puts ''\n" +
+            "    @runner_orig.run(config, options)\n" +
+            "  end\n" +
+            "end\n" +
+            "\n" +
+            "unless Embulk.const_defined?(:Runner)\n" +
+            "  Embulk.const_set :Runner, DummyEmbulkRunner.new(Embulk::EmbulkRunner.new(__internal_runner_java__))\n" +
+            "end\n");
+        globalJRubyContainer.remove("__internal_runner_java__");
     }
 
     // NOTE: The root logger directly from |LoggerFactory|, not from |Exec.getLogger| as it's outside of |Exec.doWith|.
@@ -590,9 +579,4 @@ public class EmbulkRunner
     private final Pattern EXT_YAML_LIQUID = Pattern.compile(".*\\.ya?ml\\.liquid$");
 
     private final EmbulkEmbed embed;
-    private final List<String> jrubyOptions;
-    private final List<String> loadPaths;
-    private final List<String> pluginPaths;
-    private final List<String> classpaths;
-    private final String bundlePath;
 }