embulk 0.8.23-java → 0.8.24-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e99caa00f5816bc5c6cf1e3d1f01dafbf985884a
-  data.tar.gz: 918416d9c1be50518d9a7351eddfb66b3e10963c
+  metadata.gz: 4304649d3c657ab61884394e4b04c32934ecc516
+  data.tar.gz: e99e2b6296c4660d6da94e9465e502dcdd218f22
 SHA512:
-  metadata.gz: 6fdf176ac2e1082d929cd3d376b5f6b598245b5986cc984628a2b2300dba96038592e5ff6aa3d6f0664d6b615edd4721a56c3ea6724d016b3bd0b5c019d08139
-  data.tar.gz: 4c9d2e4947621e75a47fa37f0c490feea831ee0ac2c203f4d48cd5e45ea4645d15b2414e8b110b5ddf6a894df9f20d029cb01da753c77555b58634bb2b320620
+  metadata.gz: 29a99a36b30105da946f964c8f131c98a5f3dedb2fa16edf6e50b2463f9b08939ee4602e0cf6353bde581ae6b7ee471dbb5da8a0ca6a90d6872308609c15e6ad
+  data.tar.gz: 0496df6ebb63f5bbef27d3dbf0fed1b6a2e8d6609b17651009f2c9498d97df289ca8a9e0f2f1868031b0aa273be28c36609552cc88d47bec9d4592f7cea8c9a8

data/bin/embulk

@@ -6,6 +6,33 @@ if RUBY_PLATFORM =~ /java/i
     ENV.delete('EMBULK_BIN_ENABLE_BUNDLE')
     # include -cp CLASSPATH to LOAD_PATH so that embulk_bundle.rb can load bundler included in embulk-core.jar
     $LOAD_PATH << "uri:classloader:/"
+
+    # Handle environment variables before entering embulk_bundle.rb.
+    # See: https://github.com/embulk/embulk/pull/604
+
+    bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
+    bundle_path = nil if bundle_path.empty?
+
+    # Search for -b or --bundle, and remove it.
+    if ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+      ARGV.slice!(bundle_path_index, 2)[1]
+    end
+    if bundle_path
+      ENV['EMBULK_BUNDLE_PATH'] = bundle_path
+      ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
+      ENV.delete('GEM_HOME')
+      ENV.delete('GEM_PATH')
+    else
+      ENV.delete('EMBULK_BUNDLE_PATH')
+      user_home = java.lang.System.properties["user.home"] || ENV['HOME']
+      unless user_home
+        raise "HOME environment variable is not set."
+      end
+      ENV['GEM_HOME'] = File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
+      ENV['GEM_PATH'] = ''
+      ENV.delete('BUNDLE_GEMFILE')
+    end
+
     require_relative '../lib/embulk/command/embulk_bundle'
   else
     # bin/embulk is run by JRuby (embulk gem for JRuby is installed). disable embulk_bundle not to bother the JRuby's bundler
@@ -107,4 +134,3 @@ cmdline << __FILE__
 cmdline.concat ARGV
 exec env, *cmdline
 exit 127
-

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards"), p
 
 allprojects {
     group = 'org.embulk'
-    version = '0.8.23'
+    version = '0.8.24'
 
     ext {
         jrubyVersion = '9.1.5.0'

data/embulk-cli/src/main/bat/selfrun.bat

@@ -30,6 +30,60 @@ if "%overwrite_optimize%" == "true" (
     )
 )
 
+setlocal enabledelayedexpansion
+
+set found_bundle_option=0
+
+for %%a in (%*) do (
+    if %%a == -b (
+        set found_bundle_option=1
+    ) else if %%a == --bundle (
+        set found_bundle_option=1
+    ) else if !found_bundle_option! == 1 (
+        set embulk_bundle_path=%%a
+        set found_bundle_option=2
+    )
+)
+
+endlocal && set EMBULK_BUNDLE_PATH=%embulk_bundle_path%
+
+if not defined EMBULK_BUNDLE_PATH (
+    set EMBULK_BUNDLE_PATH=
+    set GEM_PATH=""
+) else (
+    if not exist "%EMBULK_BUNDLE_PATH%\" (
+        echo Directory not found: "%EMBULK_BUNDLE_PATH%"
+        exit /b 1
+    )
+    set GEM_PATH=
+)
+
+setlocal enabledelayedexpansion
+
+if not defined EMBULK_BUNDLE_PATH (
+    set bundle_gemfile=
+) else (
+    call :get_absolute_path %EMBULK_BUNDLE_PATH%
+    set bundle_gemfile=!absolute_path!\Gemfile
+    if not exist !bundle_gemfile! (
+        echo Gemfile not found: "!bundle_gemfile!"
+        exit /b 1
+    )
+)
+
+endlocal && set BUNDLE_GEMFILE=%bundle_gemfile%
+
+setlocal enabledelayedexpansion
+
+if not defined EMBULK_BUNDLE_PATH (
+    for /f "delims=" %%w in ('java -cp %0 org.jruby.Main -e "print RbConfig::CONFIG['ruby_version']"') do set ruby_version=%%w
+    set gem_home=%USERPROFILE%\.embulk\jruby\!ruby_version!
+) else (
+    set gem_home=
+)
+
+endlocal && set GEM_HOME=%gem_home%
+
 if "%optimize%" == "true" (
     set java_args=-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC %java_args%
 ) else (
@@ -96,3 +150,7 @@ if not exist "%~1" (
 )
 set status=
 exit /b
+
+:get_absolute_path
+set absolute_path=%~f1
+exit /b

data/embulk-cli/src/main/sh/selfrun.sh

@@ -44,11 +44,50 @@ while true; do
     esac
 done
 
+embulk_args="$@"
+
+while [ $# -gt 0 ] ; do
+    case "$1" in
+        "-b" | "--bundle")
+            shift
+            EMBULK_BUNDLE_PATH="$1"
+            export EMBULK_BUNDLE_PATH
+            shift
+            break
+            ;;
+        *)
+            shift
+            ;;
+    esac
+done
+
+if test -z ${EMBULK_BUNDLE_PATH}; then
+    unset EMBULK_BUNDLE_PATH
+    unset BUNDLE_GEMFILE
+    GEM_HOME="`cd && pwd`/.embulk/jruby/`java -cp $0 org.jruby.Main -e 'print RbConfig::CONFIG["ruby_version"]'`"
+    export GEM_HOME
+    GEM_PATH=""
+    export GEM_PATH
+else
+    if test ! -d ${EMBULK_BUNDLE_PATH}; then
+        echo "Directory not found: \"${EMBULK_BUNDLE_PATH}\""
+        exit 127
+    fi
+    BUNDLE_GEMFILE="`cd ${EMBULK_BUNDLE_PATH} && pwd`/Gemfile"
+    if test ! -f ${BUNDLE_GEMFILE}; then
+        echo "Gemfile not found: \"${BUNDLE_GEMFILE}\""
+        exit 127
+    fi
+    export BUNDLE_GEMFILE
+    unset GEM_HOME
+    unset GEM_PATH
+fi
+
 if test "$overwrite_optimize" = "true" -o "$default_optimize" -a "$overwrite_optimize" != "false"; then
     java_args="-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC $java_args"
 else
     java_args="-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none $java_args"
 fi
 
-exec java $java_args -jar "$0" $jruby_args "$@"
+exec java $java_args -jar "$0" $jruby_args $embulk_args
 exit 127

data/embulk-core/src/main/java/org/embulk/exec/BulkLoader.java

@@ -16,6 +16,8 @@ import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.TaskReport;
 import org.embulk.plugin.PluginType;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.FileOutputRunner;
 import org.embulk.spi.Schema;
 import org.embulk.spi.Exec;
 import org.embulk.spi.ExecSession;
@@ -502,10 +504,24 @@ public class BulkLoader
             }
         }
 
-        plugins.getInputPlugin().cleanup(resume.getInputTaskSource(), resume.getInputSchema(),
+        final TaskSource inputTaskSource;
+        if (plugins.getInputPlugin() instanceof FileInputRunner) {
+            inputTaskSource = FileInputRunner.getFileInputTaskSource(resume.getInputTaskSource());
+        }
+        else {
+            inputTaskSource = resume.getInputTaskSource();
+        }
+        plugins.getInputPlugin().cleanup(inputTaskSource, resume.getInputSchema(),
                 resume.getInputTaskReports().size(), successfulInputTaskReports.build());
 
-        plugins.getOutputPlugin().cleanup(resume.getOutputTaskSource(), resume.getOutputSchema(),
+        final TaskSource outputTaskSource;
+        if (plugins.getOutputPlugin() instanceof FileOutputRunner) {
+            outputTaskSource = FileOutputRunner.getFileOutputTaskSource(resume.getOutputTaskSource());
+        }
+        else {
+            outputTaskSource = resume.getOutputTaskSource();
+        }
+        plugins.getOutputPlugin().cleanup(outputTaskSource, resume.getOutputSchema(),
                 resume.getOutputTaskReports().size(), successfulOutputTaskReports.build());
     }
 

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -161,4 +161,9 @@ public class FileInputRunner
             }
         }
     }
+
+    public static TaskSource getFileInputTaskSource(TaskSource runnerTaskSource)
+    {
+        return runnerTaskSource.loadTask(RunnerTask.class).getFileInputTaskSource();
+    }
 }

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -191,4 +191,9 @@ public class FileOutputRunner
             return tran.commit();
         }
     }
+
+    public static TaskSource getFileOutputTaskSource(TaskSource runnerTaskSource)
+    {
+        return runnerTaskSource.loadTask(RunnerTask.class).getFileOutputTaskSource();
+    }
 }

data/embulk-docs/src/built-in.rst

@@ -241,6 +241,9 @@ The ``columns`` option declares the list of columns. This CSV parser plugin igno
 | date     | Set date part if the format doesn’t include date part  |
 +----------+--------------------------------------------------------+
 
+.. note::
+
+   The Timestamp format refers to `Ruby strftime format <https://docs.ruby-lang.org/en/2.4.0/Date.html#method-i-strftime>`_
 
 List of types:
 
@@ -497,6 +500,10 @@ The ``column_options`` option is a map whose keys are name of columns, and value
 | format               | string  | Timestamp format if type of this column is timestamp.                                                 | ``%Y-%m-%d %H:%M:%S.%6N %z`` by default |
 +----------------------+---------+-------------------------------------------------------------------------------------------------------+-----------------------------------------+
 
+.. note::
+
+   The Timestamp format refers to `Ruby strftime format <https://docs.ruby-lang.org/en/2.4.0/Date.html#method-i-strftime>`_
+
 Example
 ~~~~~~~~
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.24
     release/release-0.8.23
     release/release-0.8.22
     release/release-0.8.21

data/embulk-docs/src/release/release-0.8.24.rst

@@ -0,0 +1,15 @@
+Release 0.8.24
+==================================
+
+General Changes
+------------------
+
+* Make environment variables immutable in JRuby/Java [#604] [#670]
+* Add a note on the timestamp format in embulk-doc with a link to Ruby documents [#648]
+* Add clean illegal characters mode to json parser. [#651]
+* embulk-core: Fix BulkLoader to pass file input/output plugins' task source to plugins' cleanup [#663]
+
+
+Release Date
+------------------
+2017-06-14

data/embulk-standards/src/main/java/org/embulk/standards/JsonParserPlugin.java

@@ -1,6 +1,10 @@
 package org.embulk.standards;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Function;
+import com.google.common.collect.Lists;
+import com.google.common.io.CharSource;
+import com.google.common.io.CharStreams;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
@@ -18,20 +22,50 @@ import org.embulk.spi.json.JsonParseException;
 import org.embulk.spi.json.JsonParser;
 import org.embulk.spi.type.Types;
 import org.embulk.spi.util.FileInputInputStream;
+import org.jruby.embed.io.ReaderInputStream;
+import org.msgpack.core.Preconditions;
 import org.msgpack.value.Value;
 import org.slf4j.Logger;
 
+import javax.annotation.Nullable;
+import java.io.BufferedReader;
 import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.regex.Pattern;
 
 public class JsonParserPlugin
         implements ParserPlugin
 {
+
+    public enum InvalidEscapeStringPolicy
+    {
+        PASSTHROUGH("PASSTHROUGH"),
+        SKIP("SKIP"),
+        UNESCAPE("UNESCAPE");
+
+        private final String string;
+
+        private InvalidEscapeStringPolicy(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString()
+        {
+            return string;
+        }
+    }
+
     public interface PluginTask
             extends Task
     {
         @Config("stop_on_invalid_record")
         @ConfigDefault("false")
         boolean getStopOnInvalidRecord();
+
+        @Config("invalid_string_escapes")
+        @ConfigDefault("\"PASSTHROUGH\"")
+        InvalidEscapeStringPolicy getInvalidEscapeStringPolicy();
     }
 
     private final Logger log;
@@ -66,7 +100,7 @@ public class JsonParserPlugin
                 FileInputInputStream in = new FileInputInputStream(input)) {
             while (in.nextFile()) {
                 boolean evenOneJsonParsed = false;
-                try (JsonParser.Stream stream = newJsonStream(in)) {
+                try (JsonParser.Stream stream = newJsonStream(in, task)) {
                     Value value;
                     while ((value = stream.next()) != null) {
                         try {
@@ -107,10 +141,87 @@ public class JsonParserPlugin
         return new PageBuilder(Exec.getBufferAllocator(), schema, output);
     }
 
-    private JsonParser.Stream newJsonStream(FileInputInputStream in)
+    private JsonParser.Stream newJsonStream(FileInputInputStream in, PluginTask task)
             throws IOException
     {
-        return new JsonParser().open(in);
+        InvalidEscapeStringPolicy policy = task.getInvalidEscapeStringPolicy();
+        switch (policy) {
+            case SKIP:
+            case UNESCAPE:
+                Iterable<CharSource> lines = Lists.transform(CharStreams.readLines(new BufferedReader(new InputStreamReader(in))),
+                        invalidEscapeStringFunction(policy));
+                return new JsonParser().open(new ReaderInputStream(CharSource.concat(lines).openStream()));
+            case PASSTHROUGH:
+            default:
+                return new JsonParser().open(in);
+        }
+    }
+
+    Function<String, CharSource> invalidEscapeStringFunction(final InvalidEscapeStringPolicy policy)
+    {
+        return new Function<String, CharSource>()
+        {
+            final Pattern digitsPattern = Pattern.compile("\\p{XDigit}+");
+
+            @Override
+            public CharSource apply(@Nullable String input)
+            {
+                Preconditions.checkNotNull(input);
+                if (policy == InvalidEscapeStringPolicy.PASSTHROUGH) {
+                    return CharSource.wrap(input);
+                }
+                StringBuilder builder = new StringBuilder();
+                char[] charArray = input.toCharArray();
+                for (int characterIndex = 0; characterIndex < charArray.length; characterIndex++) {
+                    char c = charArray[characterIndex];
+                    if (c == '\\') {
+                        if (charArray.length > characterIndex + 1) {
+                            char next = charArray[characterIndex + 1];
+                            switch (next) {
+                                case 'b':
+                                case 'f':
+                                case 'n':
+                                case 'r':
+                                case 't':
+                                case '"':
+                                case '\\':
+                                case '/':
+                                    builder.append(c);
+                                    break;
+                                case 'u': // hexstring such as \u0001
+                                    if (charArray.length > characterIndex + 5) {
+                                        char[] hexChars = {charArray[characterIndex + 2], charArray[characterIndex + 3], charArray[characterIndex + 4],
+                                                           charArray[characterIndex + 5]};
+                                        String hexString = new String(hexChars);
+                                        if (digitsPattern.matcher(hexString).matches()) {
+                                            builder.append(c);
+                                        } else {
+                                            if (policy == InvalidEscapeStringPolicy.SKIP) {
+                                                // remove \\u
+                                                characterIndex++;
+                                            }
+                                        }
+                                    }
+                                    break;
+                                default:
+                                    switch (policy) {
+                                        case SKIP:
+                                            characterIndex++;
+                                            break;
+                                        case UNESCAPE:
+                                            break;
+                                    }
+                                    break;
+                            }
+                        }
+                    }
+                    else {
+                        builder.append(c);
+                    }
+                }
+                return CharSource.wrap(builder.toString());
+            }
+        };
     }
 
     static class JsonRecordValidateException

data/embulk-standards/src/test/java/org/embulk/standards/TestJsonParserPlugin.java

@@ -1,10 +1,12 @@
 package org.embulk.standards;
 
 import com.google.common.collect.ImmutableList;
+import com.google.common.io.CharSource;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
 import org.embulk.spi.FileInput;
 import org.embulk.spi.ParserPlugin;
 import org.embulk.spi.Schema;
@@ -22,6 +24,9 @@ import java.io.InputStream;
 import java.util.List;
 import java.util.Map;
 
+import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.PASSTHROUGH;
+import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.SKIP;
+import static org.embulk.standards.JsonParserPlugin.InvalidEscapeStringPolicy.UNESCAPE;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
@@ -48,6 +53,17 @@ public class TestJsonParserPlugin
         output = new MockPageOutput();
     }
 
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource();
+
+        JsonParserPlugin.PluginTask task = config.loadConfig(JsonParserPlugin.PluginTask.class);
+        assertEquals(false, task.getStopOnInvalidRecord());
+        assertEquals(JsonParserPlugin.InvalidEscapeStringPolicy.PASSTHROUGH, task.getInvalidEscapeStringPolicy());
+    }
+
+
     @Test
     public void readNormalJson()
             throws Exception
@@ -132,6 +148,172 @@ public class TestJsonParserPlugin
         }
     }
 
+    @Test
+    public void useDefaultInvalidEscapeStringFunction()
+            throws Exception
+    {
+        try {
+            transaction(config, fileInput(
+                    "{\"\\a\":\"b\"}\\" // throw DataException
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void usePassthroughInvalidEscapeStringFunction()
+            throws Exception
+    {
+        try {
+            ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "PASSTHROUGH");
+            transaction(config, fileInput(
+                    "{\"\\a\":\"b\"}\\" // throw DataException
+            ));
+            fail();
+        }
+        catch (Throwable t) {
+            assertTrue(t instanceof DataException);
+        }
+    }
+
+    @Test
+    public void useSkipInvalidEscapeString()
+            throws Exception
+    {
+        ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "SKIP");
+        transaction(config, fileInput(
+                "{\"\\a\":\"b\"}\\"
+        ));
+
+        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
+        assertEquals(1, records.size());
+        Object[] record = records.get(0);
+        Map<Value, Value> map = ((Value)record[0]).asMapValue().map();
+        assertEquals(newString("b"), map.get(newString("")));
+    }
+
+    @Test
+    public void useUnEscapeInvalidEscapeString()
+            throws Exception
+    {
+        ConfigSource config = this.config.deepCopy().set("invalid_string_escapes", "UNESCAPE");
+        transaction(config, fileInput(
+                "{\"\\a\":\"b\"}\\"
+        ));
+
+        List<Object[]> records = Pages.toObjects(plugin.newSchema(), output.pages);
+        assertEquals(1, records.size());
+        Object[] record = records.get(0);
+        Map<Value, Value> map = ((Value)record[0]).asMapValue().map();
+        assertEquals(newString("b"), map.get(newString("a")));
+    }
+
+    @Test
+    public void checkInvalidEscapeStringFunction()
+            throws Exception
+    {
+        //PASSTHROUGH
+        {
+            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
+            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
+            assertEquals(json , actual.read());
+        }
+
+        {
+            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
+            assertEquals(json , actual.read());
+        }
+
+        {
+            String json = "{\"\\a\":\"b\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(PASSTHROUGH).apply(json);
+            assertEquals(json , actual.read());
+        }
+
+        //SKIP
+        {
+            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            assertEquals(json , actual.read());
+        }
+
+        {
+            // valid charset u0001
+            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            assertEquals("{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}" , actual.read());
+        }
+
+
+        {
+            // invalid charset \\u12xY remove forwarding backslash and u
+            String json = "{\"\\u12xY\":\"efg\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            assertEquals("{\"12xY\":\"efg\"}" , actual.read());
+        }
+
+        {
+            String json = "{\"\\a\":\"b\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            // backslash and `a` will removed.
+            assertEquals("{\"\":\"b\"}" , actual.read());
+        }
+
+        {
+            // end of lines backspash.
+            String json = "{\"\\a\":\"b\"}" +
+                    "\n" +
+                    "\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            // backslash and `a` will removed.
+            assertEquals("{\"\":\"b\"}\n" , actual.read());
+        }
+
+        //UNESCAPE
+        {
+            String json = "{\\\"_c0\\\":true,\\\"_c1\\\":10,\\\"_c2\\\":\\\"embulk\\\",\\\"_c3\\\":{\\\"k\\\":\\\"v\\\"}}";
+            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
+            assertEquals(json , actual.read());
+        }
+
+        {
+            String json = "{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
+            assertEquals("{\"abc\b\f\n\r\t\\\\u0001\":\"efg\"}" , actual.read());
+        }
+
+        {
+            // invalid charset u000x remove forwarding backslash
+            String json = "{\"\\u000x\":\"efg\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
+            assertEquals("{\"u000x\":\"efg\"}" , actual.read());
+        }
+
+
+        {
+            String json = "{\"\\a\":\"b\"}\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(UNESCAPE).apply(json);
+            // backslash will removed.
+            assertEquals("{\"a\":\"b\"}" , actual.read());
+        }
+
+        {
+            // end of lines backspash.
+            String json = "{\"\\a\":\"b\"}" +
+                    "\n" +
+                    "\\";
+            CharSource actual = plugin.invalidEscapeStringFunction(SKIP).apply(json);
+            // backslash and `a` will removed.
+            assertEquals("{\"\":\"b\"}\n" , actual.read());
+        }
+
+
+    }
+
     private ConfigSource config()
     {
         return runtime.getExec().newConfigSource();

data/lib/embulk/command/embulk_bundle.rb

@@ -1,20 +1,19 @@
-
 bundle_path = ENV['EMBULK_BUNDLE_PATH'].to_s
 bundle_path = nil if bundle_path.empty?
 
-# search -b, --bundle BUNDLE_DIR
-bundle_path_index = ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
-if bundle_path_index
-  bundle_path = ARGV.slice!(bundle_path_index, 2)[1]
+# Search for -b or --bundle, and remove it.
+if ARGV.find_index {|arg| arg == '-b' || arg == '--bundle' }
+  ARGV.slice!(bundle_path_index, 2)[1]
 end
 
 if bundle_path
-  ENV['EMBULK_BUNDLE_PATH'] = bundle_path
-  ENV['BUNDLE_GEMFILE'] = File.expand_path File.join(bundle_path, "Gemfile")
+  # In the selfrun script:
+  # ENV['EMBULK_BUNDLE_PATH']: set through '-b' | '--bundle', or inherit from the runtime environment
+  # ENV['BUNDLE_GEMFILE']: set for "ENV['EMBULK_BUNDLE_PATH']/Gemfile"
+  # ENV['GEM_HOME']: unset
+  # ENV['GEM_PATH']: unset
 
   # bundler is included in embulk-core.jar
-  ENV.delete('GEM_HOME')
-  ENV.delete('GEM_PATH')
   Gem.clear_paths
   require 'bundler'
 
@@ -34,17 +33,12 @@ if bundle_path
   end
 
 else
-  # default GEM_HOME is ~/.embulk/jruby/1.9/. If -b option is set,
-  # GEM_HOME is already set by embulk/command/embulk_main.rb
-  ENV.delete('EMBULK_BUNDLE_PATH')
-  user_home = java.lang.System.properties["user.home"] || ENV['HOME']
-  unless user_home
-    raise "HOME environment variable is not set."
-  end
-  ENV['GEM_HOME'] = File.expand_path File.join(user_home, '.embulk', Gem.ruby_engine, RbConfig::CONFIG['ruby_version'])
-  ENV['GEM_PATH'] = ''
+  # In the selfrun script:
+  # ENV['EMBULK_BUNDLE_PATH']: unset
+  # ENV['BUNDLE_GEMFILE']: unset
+  # ENV['GEM_HOME']: set for "~/.embulk/jruby/${ruby-version}"
+  # ENV['GEM_PATH']: set for ""
 
-  ENV.delete('BUNDLE_GEMFILE')
   Gem.clear_paths  # force rubygems to reload GEM_HOME
 
   $LOAD_PATH << File.expand_path('../../', File.dirname(__FILE__))

data/lib/embulk/version.rb

@@ -3,7 +3,7 @@
 module Embulk
   @@warned = false
 
-  VERSION_INTERNAL = '0.8.23'
+  VERSION_INTERNAL = '0.8.24'
 
   DEPRECATED_MESSAGE = 'Embulk::VERSION in (J)Ruby is deprecated. Use org.embulk.EmbulkVersion::VERSION instead. If this message is from a plugin, please tell this to the author of the plugin!'
   def self.const_missing(name)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.8.23
+  version: 0.8.24
 platform: java
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-06-06 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -150,9 +150,9 @@ files:
 - classpath/commons-compress-1.10.jar
 - classpath/commons-lang-2.4.jar
 - classpath/commons-lang3-3.4.jar
-- classpath/embulk-cli-0.8.23.jar
-- classpath/embulk-core-0.8.23.jar
-- classpath/embulk-standards-0.8.23.jar
+- classpath/embulk-cli-0.8.24.jar
+- classpath/embulk-core-0.8.24.jar
+- classpath/embulk-standards-0.8.24.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-bootstrap-0.1.1.jar
@@ -502,6 +502,7 @@ files:
 - embulk-docs/src/release/release-0.8.21.rst
 - embulk-docs/src/release/release-0.8.22.rst
 - embulk-docs/src/release/release-0.8.23.rst
+- embulk-docs/src/release/release-0.8.24.rst
 - embulk-docs/src/release/release-0.8.3.rst
 - embulk-docs/src/release/release-0.8.4.rst
 - embulk-docs/src/release/release-0.8.5.rst