embulk 0.8.31 → 0.8.32

data/embulk-core/src/main/java/org/embulk/EmbulkSetup.java

@@ -16,13 +16,7 @@ import org.embulk.config.ConfigSource;
  */
 public class EmbulkSetup
 {
-    public static EmbulkRunner setup(
-            final List<String> jrubyOptions,
-            final Map<String, Object> systemConfigGiven,
-            final List<String> loadPaths,
-            final List<String> pluginPaths,
-            final List<String> classpaths,
-            final String bundlePath)
+    public static EmbulkRunner setup(final Map<String, Object> systemConfigGiven)
     {
         // NOTE: When it was in Ruby "require 'json'" was required to format the system config into a JSON string.
 
@@ -50,6 +44,6 @@ public class EmbulkSetup
         bootstrap.setSystemConfig(systemConfig);
         final EmbulkEmbed embed = bootstrap.initialize();  // see embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.
 
-        return new EmbulkRunner(embed, jrubyOptions, loadPaths, pluginPaths, classpaths, bundlePath);
+        return new EmbulkRunner(embed);
     }
 }

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -1,8 +1,17 @@
 package org.embulk.jruby;
 
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.CodeSource;
+import java.security.ProtectionDomain;
 import java.util.List;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.Set;
+import org.slf4j.Logger;
 import org.slf4j.ILoggerFactory;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Module;
@@ -14,8 +23,11 @@ import com.google.inject.Injector;
 import com.google.inject.Key;
 import com.google.inject.spi.Dependency;
 import com.google.inject.spi.ProviderWithDependencies;
+import org.jruby.RubyInstanceConfig;
 import org.jruby.embed.LocalContextScope;
+import org.jruby.embed.LocalVariableBehavior;
 import org.jruby.embed.ScriptingContainer;
+import org.jruby.util.cli.Options;
 import org.embulk.plugin.PluginSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ModelManager;
@@ -44,6 +56,10 @@ public class JRubyScriptingModule
         private final Injector injector;
         private final boolean useGlobalRubyRuntime;
         private final String gemHome;
+        private final List<String> jrubyClasspath;
+        private final List<String> jrubyLoadPath;
+        private final List<String> jrubyOptions;
+        private final String jrubyBundlerPluginSourceDirectory;
 
         @Inject
         public ScriptingContainerProvider(Injector injector, @ForSystemConfig ConfigSource systemConfig)
@@ -57,13 +73,101 @@ public class JRubyScriptingModule
             this.gemHome = systemConfig.get(String.class, "gem_home", null);
 
             // TODO get jruby-home from systemConfig to call jruby.container.setHomeDirectory
-            // TODO get jruby-load-paths from systemConfig to call jruby.container.setLoadPaths
+
+            final List jrubyLoadPathNonGeneric = systemConfig.get(List.class, "jruby_load_path", null);
+            final ArrayList<String> jrubyLoadPathBuilt = new ArrayList<String>();
+            if (jrubyLoadPathNonGeneric != null) {
+                for (final Object oneJRubyLoadPath : jrubyLoadPathNonGeneric) {
+                    if (oneJRubyLoadPath instanceof String) {
+                        jrubyLoadPathBuilt.add((String) oneJRubyLoadPath);
+                    }
+                    else {
+                        // It should happen only in very irregular cases. Okay to create |Logger| every time.
+                        Logger logger = injector.getInstance(ILoggerFactory.class).getLogger("init");
+                        logger.warn("System config \"jruby_load_path\" contains non-String.");
+                        jrubyLoadPathBuilt.add(oneJRubyLoadPath.toString());
+                    }
+                }
+            }
+            this.jrubyLoadPath = Collections.unmodifiableList(jrubyLoadPathBuilt);
+
+            final List jrubyClasspathNonGeneric = systemConfig.get(List.class, "jruby_classpath", new ArrayList());
+            final ArrayList<String> jrubyClasspathBuilt = new ArrayList<String>();
+            if (jrubyClasspathNonGeneric != null) {
+                for (final Object oneJRubyClasspath : jrubyClasspathNonGeneric) {
+                    if (oneJRubyClasspath instanceof String) {
+                        jrubyClasspathBuilt.add((String) oneJRubyClasspath);
+                    }
+                    else {
+                        // It should happen only in very irregular cases. Okay to create |Logger| every time.
+                        Logger logger = injector.getInstance(ILoggerFactory.class).getLogger("init");
+                        logger.warn("System config \"jruby_classpath\" contains non-String.");
+                        jrubyClasspathBuilt.add(oneJRubyClasspath.toString());
+                    }
+                }
+            }
+            this.jrubyClasspath = Collections.unmodifiableList(jrubyClasspathBuilt);
+
+            final List jrubyOptionsNonGeneric = systemConfig.get(List.class, "jruby_command_line_options", null);
+            final ArrayList<String> jrubyOptionsBuilt = new ArrayList<String>();
+            if (jrubyOptionsNonGeneric != null) {
+                for (final Object oneJRubyOption : jrubyOptionsNonGeneric) {
+                    if (oneJRubyOption instanceof String) {
+                        jrubyOptionsBuilt.add((String) oneJRubyOption);
+                    }
+                    else {
+                        // It should happen only in very irregular cases. Okay to create |Logger| every time.
+                        Logger logger = injector.getInstance(ILoggerFactory.class).getLogger("init");
+                        logger.warn("System config \"jruby_command_line_options\" contains non-String.");
+                        jrubyOptionsBuilt.add(oneJRubyOption.toString());
+                    }
+                }
+            }
+            this.jrubyOptions = Collections.unmodifiableList(jrubyOptionsBuilt);
+
+            this.jrubyBundlerPluginSourceDirectory =
+                systemConfig.get(String.class, "jruby_global_bundler_plugin_source_directory", null);
         }
 
         public ScriptingContainer get()
         {
             LocalContextScope scope = (useGlobalRubyRuntime ? LocalContextScope.SINGLETON : LocalContextScope.SINGLETHREAD);
-            ScriptingContainer jruby = new ScriptingContainer(scope);
+            ScriptingContainer jruby = new ScriptingContainer(scope, LocalVariableBehavior.PERSISTENT);
+            final RubyInstanceConfig jrubyInstanceConfig = jruby.getProvider().getRubyInstanceConfig();
+            for (final String jrubyOption : this.jrubyOptions) {
+                try {
+                    processJRubyOption(jrubyOption, jrubyInstanceConfig);
+                }
+                catch (UnrecognizedJRubyOptionException ex) {
+                    final Logger logger = this.injector.getInstance(ILoggerFactory.class).getLogger("init");
+                    logger.error("The \"-R\" option(s) are not recognized in Embulk: -R" + jrubyOption +
+                                 ". Please add your requests at: https://github.com/embulk/embulk/issues/707", ex);
+                    throw new RuntimeException(ex);
+
+                }
+                catch (NotWorkingJRubyOptionException ex) {
+                    final Logger logger = this.injector.getInstance(ILoggerFactory.class).getLogger("init");
+                    logger.warn("The \"-R\" option(s) do not work in Embulk: -R" + jrubyOption + ".", ex);
+                }
+            }
+
+            setBundlerPluginSourceDirectory(jruby, this.jrubyBundlerPluginSourceDirectory);
+
+            for (final String oneJRubyLoadPath : this.jrubyLoadPath) {
+                // ruby script directory (use unshift to make it highest priority)
+                jruby.put("__internal_load_path__", oneJRubyLoadPath);
+                // TODO: Check if $LOAD_PATH already contains it.
+                jruby.runScriptlet("$LOAD_PATH.unshift File.expand_path(__internal_load_path__)");
+                jruby.remove("__internal_load_path__");
+            }
+
+            for (final String oneJRubyClasspath : this.jrubyClasspath) {
+                jruby.put("__internal_classpath__", oneJRubyClasspath);
+                // $CLASSPATH object doesn't have concat method
+                // TODO: Check if $CLASSPATH already contains it.
+                jruby.runScriptlet("$CLASSPATH << __internal_classpath__");
+                jruby.remove("__internal_classpath__");
+            }
 
             // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
             // $LOAD_PATH is set by lib/embulk/command/embulk_run.rb if Embulk starts
@@ -128,5 +232,158 @@ public class JRubyScriptingModule
                 Dependency.get(Key.get(ModelManager.class)),
                 Dependency.get(Key.get(BufferAllocator.class)));
         }
+
+        private static final class UnrecognizedJRubyOptionException extends Exception {}
+        private static final class NotWorkingJRubyOptionException extends Exception {}
+        private static final class UnrecognizedJRubyLoadPathException extends Exception {
+            public UnrecognizedJRubyLoadPathException(final String message)
+            {
+                super(message);
+            }
+
+            public UnrecognizedJRubyLoadPathException(final String message, final Throwable cause)
+            {
+                super(message, cause);
+            }
+        }
+
+        private void setBundlerPluginSourceDirectory(final ScriptingContainer jruby, final String directory)
+        {
+            if (directory != null) {
+                /* Environment variables are set in the selfrun script or bin/embulk:
+                 *   ENV['EMBULK_BUNDLE_PATH']: set through '-b' | '--bundle', or inherit from the runtime environment
+                 *   ENV['BUNDLE_GEMFILE']: set for "ENV['EMBULK_BUNDLE_PATH']/Gemfile"
+                 *   ENV['GEM_HOME']: unset
+                 *   ENV['GEM_PATH']: unset
+                 */
+
+                // bundler is included in embulk-core.jar
+                jruby.runScriptlet("Gem.clear_paths");
+                jruby.runScriptlet("require 'bundler'");
+
+                jruby.runScriptlet("Bundler.load.setup_environment");
+                jruby.runScriptlet("require 'bundler/setup'");
+                // since here, `require` may load files of different (newer) embulk versions
+                // especially following 'embulk/command/embulk_main'.
+
+                // NOTE: It is intentionally not done by building a Ruby statement string from |directory|.
+                // It can cause insecure injections.
+                //
+                // add bundle directory path to load local plugins at ./embulk
+                jruby.put("__internal_bundler_plugin_source_directory__", directory);
+                jruby.runScriptlet("$LOAD_PATH << File.expand_path(__internal_bundler_plugin_source_directory__)");
+                jruby.remove("__internal_bundler_plugin_source_directory__");
+            }
+            else {
+                /* Environment variables are set in the selfrun script or bin/embulk:
+                 *   ENV['EMBULK_BUNDLE_PATH']: unset
+                 *   ENV['BUNDLE_GEMFILE']: unset
+                 *   ENV['GEM_HOME']: set for "~/.embulk/jruby/${ruby-version}"
+                 *   ENV['GEM_PATH']: set for ""
+                 */
+
+                jruby.runScriptlet("Gem.clear_paths");  // force rubygems to reload GEM_HOME
+
+                // NOTE: The path from |buildJRubyLoadPath()| is added in $LOAD_PATH just in case.
+                // Though it is not mandatory just to run "embulk_main.rb", it may be required in later steps.
+                //
+                // NOTE: It is intentionally not done by building a Ruby statement string from |buildJRubyLoadPath()|.
+                // It can cause insecure injections.
+                //
+                // NOTE: It was written in Ruby as follows:
+                //   $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))
+                final String jrubyLoadPath;
+                try {
+                    jrubyLoadPath = buildJRubyLoadPath();
+                }
+                catch (UnrecognizedJRubyLoadPathException ex) {
+                    final Logger logger = this.injector.getInstance(ILoggerFactory.class).getLogger("init");
+                    logger.error("Failed to retrieve Embulk's location.", ex);
+                    throw new RuntimeException(ex);
+                }
+                jruby.put("__internal_load_path__", jrubyLoadPath);
+                jruby.runScriptlet("$LOAD_PATH << File.expand_path(__internal_load_path__)");
+                jruby.remove("__internal_load_path__");
+            }
+        }
+
+        private static void processJRubyOption(final String jrubyOption, final RubyInstanceConfig jrubyInstanceConfig)
+                throws UnrecognizedJRubyOptionException, NotWorkingJRubyOptionException
+        {
+            if (jrubyOption.charAt(0) != '-') {
+                throw new UnrecognizedJRubyOptionException();
+            }
+
+            for (int index = 1; index < jrubyOption.length(); ++index) {
+                switch (jrubyOption.charAt(index)) {
+                case '-':
+                    if (jrubyOption.equals("--dev")) {
+                        // They are not all of "--dev", but they are most possible configurations after JVM boot.
+                        Options.COMPILE_INVOKEDYNAMIC.force("false");  // NOTE: Options is global.
+                        jrubyInstanceConfig.setCompileMode(RubyInstanceConfig.CompileMode.OFF);
+                        return;
+                    }
+                    else if (jrubyOption.equals("--client")) {
+                        throw new NotWorkingJRubyOptionException();
+                    }
+                    else if (jrubyOption.equals("--server")) {
+                        throw new NotWorkingJRubyOptionException();
+                    }
+                    throw new UnrecognizedJRubyOptionException();
+                default:
+                    throw new UnrecognizedJRubyOptionException();
+                }
+            }
+        }
+
+        /**
+         * Returns a path to be added in JRuby's $LOAD_PATH.
+         *
+         * In case Embulk runs from the Embulk JAR file (normal case):
+         *     "file:/some/directory/embulk.jar!"
+         *
+         * In case Embulk runs out of a JAR file (irregular case):
+         *     "/some/directory"
+         */
+        private static String buildJRubyLoadPath()
+                throws UnrecognizedJRubyLoadPathException
+        {
+            final ProtectionDomain protectionDomain;
+            try {
+                protectionDomain = JRubyScriptingModule.class.getProtectionDomain();
+            }
+            catch (SecurityException ex) {
+                throw new UnrecognizedJRubyLoadPathException("Failed to achieve ProtectionDomain", ex);
+            }
+
+            final CodeSource codeSource = protectionDomain.getCodeSource();
+            if (codeSource == null) {
+                throw new UnrecognizedJRubyLoadPathException("Failed to achieve CodeSource");
+            }
+
+            final URL locationUrl = codeSource.getLocation();
+            if (locationUrl == null) {
+                throw new UnrecognizedJRubyLoadPathException("Failed to achieve location");
+            }
+            else if (!locationUrl.getProtocol().equals("file")) {
+                throw new UnrecognizedJRubyLoadPathException("Invalid location: " + locationUrl.toString());
+            }
+
+            final Path locationPath;
+            try {
+                locationPath = Paths.get(locationUrl.toURI());
+            }
+            catch (URISyntaxException ex) {
+                throw new UnrecognizedJRubyLoadPathException("Invalid location: " + locationUrl.toString(), ex);
+            }
+
+            if (Files.isDirectory(locationPath)) {  // Out of a JAR file
+                System.err.println("[WARN] Embulk looks running out of the Embulk jar file. It is unsupported.");
+                return locationPath.toString();
+            }
+
+            // TODO: Consider checking the file is really a JAR file.
+            return locationUrl.toString() + "!";  // Inside the Embulk JAR file
+        }
     }
 }

data/embulk-core/src/main/java/org/embulk/spi/Page.java

@@ -3,6 +3,22 @@ package org.embulk.spi;
 import java.util.List;
 import org.msgpack.value.ImmutableValue;
 
+/**
+ * Page is an in-process (in-JVM) container of data records.
+ *
+ * It serializes records to byte[] (in org.embulk.spi.Buffer) in order to:
+ * A) Avoid slowness by handling many Java Objects
+ * B) Avoid complexity by type-safe primitive arrays
+ * C) Track memory consumption by records
+ * D) Use off-heap memory
+ *
+ * (C) and (D) may not be so meaningful as of v0.7+ (or since earlier) as recent Embulk unlikely
+ * allocates so many Pages at the same time. Recent Embulk is streaming-driven instead of
+ * multithreaded queue-based.
+ *
+ * Page is NOT for inter-process communication. For multi-process execution such as MapReduce
+ * Executor, the executor plugin takes responsibility about interoperable serialization.
+ */
 public class Page
 {
     private final Buffer buffer;

data/embulk-core/src/test/java/org/embulk/spi/json/TestJsonParser.java

@@ -0,0 +1,102 @@
+package org.embulk.spi.json;
+
+import org.junit.Test;
+import org.msgpack.value.Value;
+import org.msgpack.value.ValueType;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestJsonParser
+{
+    @Test
+    public void testString() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("\"foobar\"");
+        assertFalse(msgpackValue.getValueType().isNumberType());
+        assertTrue(msgpackValue.getValueType().isStringType());
+        assertEquals("foobar", msgpackValue.asStringValue().toString());
+    }
+
+    @Test(expected = JsonParseException.class)
+    public void testStringUnquoted() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        parser.parse("foobar");
+    }
+
+    @Test
+    public void testOrdinaryInteger() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("12345");
+        assertTrue(msgpackValue.getValueType().isNumberType());
+        assertTrue(msgpackValue.getValueType().isIntegerType());
+        assertFalse(msgpackValue.getValueType().isFloatType());
+        assertFalse(msgpackValue.getValueType().isStringType());
+        assertEquals(12345, msgpackValue.asIntegerValue().asInt());
+    }
+
+    @Test
+    public void testExponentialInteger1() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("12345e3");
+        assertTrue(msgpackValue.getValueType().isNumberType());
+        // TODO: Consider this needs to be an integer?
+        // See: https://github.com/embulk/embulk/issues/775
+        assertTrue(msgpackValue.getValueType().isFloatType());
+        assertFalse(msgpackValue.getValueType().isIntegerType());
+        assertFalse(msgpackValue.getValueType().isStringType());
+        assertEquals(12345000.0, msgpackValue.asFloatValue().toDouble(), 0.000000001);
+        // Not sure this |toString| is to be tested...
+        assertEquals("1.2345E7", msgpackValue.asFloatValue().toString());
+    }
+
+    @Test
+    public void testExponentialInteger2() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("123e2");
+        assertTrue(msgpackValue.getValueType().isNumberType());
+        // TODO: Consider this needs to be an integer?
+        // See: https://github.com/embulk/embulk/issues/775
+        assertTrue(msgpackValue.getValueType().isFloatType());
+        assertFalse(msgpackValue.getValueType().isIntegerType());
+        assertFalse(msgpackValue.getValueType().isStringType());
+        assertEquals(12300.0, msgpackValue.asFloatValue().toDouble(), 0.000000001);
+        // Not sure this |toString| is to be tested...
+        assertEquals("12300.0", msgpackValue.asFloatValue().toString());
+    }
+
+    @Test
+    public void testOrdinaryFloat() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("12345.12");
+        assertTrue(msgpackValue.getValueType().isNumberType());
+        assertTrue(msgpackValue.getValueType().isFloatType());
+        assertFalse(msgpackValue.getValueType().isIntegerType());
+        assertFalse(msgpackValue.getValueType().isStringType());
+        assertEquals(12345.12, msgpackValue.asFloatValue().toDouble(), 0.000000001);
+        // Not sure this |toString| is to be tested...
+        assertEquals("12345.12", msgpackValue.asFloatValue().toString());
+    }
+
+    @Test
+    public void testExponentialFloat() throws Exception
+    {
+        final JsonParser parser = new JsonParser();
+        final Value msgpackValue = parser.parse("1.234512E4");
+        assertTrue(msgpackValue.getValueType().isNumberType());
+        assertTrue(msgpackValue.getValueType().isFloatType());
+        assertFalse(msgpackValue.getValueType().isIntegerType());
+        assertFalse(msgpackValue.getValueType().isStringType());
+        assertEquals(12345.12, msgpackValue.asFloatValue().toDouble(), 0.000000001);
+        // Not sure this |toString| is to be tested...
+        assertEquals("12345.12", msgpackValue.asFloatValue().toString());
+    }
+}