embulk 0.7.4 → 0.7.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd162b183e5e14cb42b36d7bed695fe530ff41a1
-  data.tar.gz: 86f9a9b4d6913adbf9cc758781e5dab4bf793c25
+  metadata.gz: 0fc04e8ab92c677b7c5891a2818354f81da3ef70
+  data.tar.gz: 4a43292bb55fec0d3d981089572e574dcafa525d
 SHA512:
-  metadata.gz: ff68ce1bb8e18e10e7e171a48b19c18cac2c52c9759d1db2116ec0180f7360b33432e1e78fe1e80717c229afc253852010b4fbc2ddcf1406ea8613b97fbde1de
-  data.tar.gz: fa257fa73c6f538e4ea390adf8d9a276404333ed787da9496e19ce9b007eb0d790186ebd4dbb1ada64afdf4a245590a6b7f268100e539f1ac9879c7898ab77c5
+  metadata.gz: 122d1f4632210c5d807d3301fdab9046a509420e9557caff224696f35a1f21eac9731dade238044fd36f70e0b24e382b276e3da4165f680d76875a8a7d399440
+  data.tar.gz: 6a671967d0c53640f977e286d0dd874712454af12ba361a7d6db85599be510da2bfdd67bc685803e0d5426b8b8b7f17e2e01c1aa3172f23c55635b8e1af8697e

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    embulk (0.7.0)
+    embulk (0.7.4)
       jruby-jars (= 9.0.0.0)
 
 GEM

data/README.md

@@ -129,6 +129,13 @@ Following command updates embulk itself to the latest released version.
 embulk selfupdate
 ```
 
+Following command updates embulk itself to the specific released version.
+
+```sh
+embulk selfupdate x.y.z
+```
+
+
 ## Embulk Development
 
 ### Build

data/build.gradle

@@ -16,7 +16,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.7.4'
+    version = '0.7.5'
 
     ext {
         jrubyVersion = '9.0.0.0'
@@ -88,10 +88,11 @@ subprojects {
             jcenter()
         }
 
-        compileJava.options.encoding = 'UTF-8' // source encoding
         sourceCompatibility = 1.7
         targetCompatibility = 1.7
 
+        def defaultEncoding = 'UTF-8'
+        [compileJava, compileTestJava]*.options*.encoding = defaultEncoding
         dependencies {
             testCompile 'junit:junit:4.12'
         }

data/embulk-core/src/main/java/org/embulk/EmbulkEmbed.java

@@ -224,7 +224,7 @@ public class EmbulkEmbed
 
     private ExecSession newExecSession(ConfigSource config)
     {
-        ConfigSource execConfig = config.deepCopy().getNestedOrSetEmpty("exec");
+        ConfigSource execConfig = config.deepCopy().getNestedOrGetEmpty("exec");
         return ExecSession.builder(injector).fromExecConfig(execConfig).build();
     }
 

data/embulk-core/src/main/java/org/embulk/config/CommitReport.java

@@ -13,6 +13,9 @@ public interface CommitReport
     @Override
     CommitReport getNestedOrSetEmpty(String attrName);
 
+    @Override
+    CommitReport getNestedOrGetEmpty(String attrName);
+
     @Override
     CommitReport set(String attrName, Object v);
 

data/embulk-core/src/main/java/org/embulk/config/ConfigDiff.java

@@ -9,6 +9,9 @@ public interface ConfigDiff
     @Override
     ConfigDiff getNestedOrSetEmpty(String attrName);
 
+    @Override
+    ConfigDiff getNestedOrGetEmpty(String attrName);
+
     @Override
     ConfigDiff set(String attrName, Object v);
 

data/embulk-core/src/main/java/org/embulk/config/ConfigSource.java

@@ -11,6 +11,9 @@ public interface ConfigSource
     @Override
     ConfigSource getNestedOrSetEmpty(String attrName);
 
+    @Override
+    ConfigSource getNestedOrGetEmpty(String attrName);
+
     @Override
     ConfigSource set(String attrName, Object v);
 

data/embulk-core/src/main/java/org/embulk/config/DataSource.java

@@ -23,6 +23,8 @@ public interface DataSource
 
     DataSource getNestedOrSetEmpty(String attrName);
 
+    DataSource getNestedOrGetEmpty(String attrName);
+
     DataSource set(String attrName, Object v);
 
     DataSource setNested(String attrName, DataSource v);

data/embulk-core/src/main/java/org/embulk/config/DataSourceImpl.java

@@ -114,6 +114,18 @@ public class DataSourceImpl
         return newInstance(model, (ObjectNode) json);
     }
 
+    @Override
+    public DataSourceImpl getNestedOrGetEmpty(String attrName)
+    {
+        JsonNode json = data.get(attrName);
+        if (json == null) {
+            json = data.objectNode();
+        } else if (!json.isObject()) {
+            throw new ConfigException("Attribute "+attrName+" must be an object");
+        }
+        return newInstance(model, (ObjectNode) json);
+    }
+
     @Override
     public DataSourceImpl set(String attrName, Object v)
     {

data/embulk-core/src/main/java/org/embulk/config/TaskReport.java

@@ -9,6 +9,9 @@ public interface TaskReport
     @Override
     TaskReport getNestedOrSetEmpty(String attrName);
 
+    @Override
+    TaskReport getNestedOrGetEmpty(String attrName);
+
     @Override
     TaskReport set(String attrName, Object v);
 

data/embulk-core/src/main/java/org/embulk/config/TaskSource.java

@@ -11,6 +11,9 @@ public interface TaskSource
     @Override
     TaskSource getNestedOrSetEmpty(String attrName);
 
+    @Override
+    TaskSource getNestedOrGetEmpty(String attrName);
+
     @Override
     TaskSource set(String attrName, Object v);
 

data/embulk-core/src/main/java/org/embulk/exec/ConfigurableGuessInputPlugin.java

@@ -0,0 +1,9 @@
+package org.embulk.exec;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+
+public interface ConfigurableGuessInputPlugin
+{
+    ConfigDiff guess(ConfigSource execConfig, ConfigSource config);
+}

data/embulk-core/src/main/java/org/embulk/exec/GuessExecutor.java

@@ -98,15 +98,21 @@ public class GuessExecutor
     private ConfigDiff doGuess(ConfigSource config)
     {
         ConfigSource inputConfig = config.getNested("in");
+        ConfigSource execConfig = config.getNestedOrGetEmpty("exec");
 
         InputPlugin input = newInputPlugin(inputConfig);
 
         ConfigDiff inputGuessed;
-        try {
-            inputGuessed = input.guess(inputConfig);
-        } catch (AbstractMethodError ex) {
-            // for backward compatibility with embulk v0.4 interface
-            throw new UnsupportedOperationException(input.getClass().getSimpleName()+".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
+        if (input instanceof ConfigurableGuessInputPlugin) {
+            inputGuessed = ((ConfigurableGuessInputPlugin) input).guess(execConfig, inputConfig);
+        }
+        else {
+            try {
+                inputGuessed = input.guess(inputConfig);
+            } catch (AbstractMethodError ex) {
+                // for backward compatibility with embulk v0.4 interface
+                throw new UnsupportedOperationException(input.getClass().getSimpleName()+".guess(ConfigSource) is not implemented. This input plugin does not support guessing.");
+            }
         }
 
         ConfigDiff wrapped = Exec.newConfigDiff();

data/embulk-core/src/main/java/org/embulk/jruby/JRubyScriptingModule.java

@@ -62,7 +62,6 @@ public class JRubyScriptingModule
         {
             LocalContextScope scope = (useGlobalRubyRuntime ? LocalContextScope.SINGLETON : LocalContextScope.SINGLETHREAD);
             ScriptingContainer jruby = new ScriptingContainer(scope);
-            jruby.setCompatVersion(CompatVersion.RUBY1_9);
 
             // Search embulk/java/bootstrap.rb from a $LOAD_PATH.
             // $LOAD_PATH is set by lib/embulk/command/embulk_run.rb if Embulk starts

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -14,10 +14,11 @@ import org.embulk.plugin.compat.PluginWrappers;
 import org.embulk.spi.util.Decoders;
 import org.embulk.exec.GuessExecutor;
 import org.embulk.exec.SamplingParserPlugin;
+import org.embulk.exec.ConfigurableGuessInputPlugin;
 import org.embulk.exec.NoSampleException;
 
 public class FileInputRunner
-        implements InputPlugin
+        implements InputPlugin, ConfigurableGuessInputPlugin
 {
     private final FileInputPlugin fileInputPlugin;
 
@@ -75,6 +76,11 @@ public class FileInputRunner
 
     @Override
     public ConfigDiff guess(ConfigSource config)
+    {
+        return guess(Exec.newConfigSource(), config);
+    }
+
+    public ConfigDiff guess(ConfigSource execConfig, ConfigSource config)
     {
         Buffer sample = SamplingParserPlugin.runFileInputSampling(this, config);
         if (sample.limit() == 0) {
@@ -82,7 +88,7 @@ public class FileInputRunner
         }
 
         GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
-        return guessExecutor.guessParserConfig(sample, config, Exec.session().getSessionExecConfig());
+        return guessExecutor.guessParserConfig(sample, config, execConfig);
     }
 
     private class RunnerControl

data/embulk-core/src/main/java/org/embulk/spi/time/Timestamp.java

@@ -11,10 +11,10 @@ import org.jruby.RubyTime;
 public class Timestamp
         implements Comparable<Timestamp>
 {
-    private final static DateTimeFormatter TO_STRING_FORMATTER_SECONDS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss z").withZoneUTC();
-    private final static DateTimeFormatter TO_STRING_FORMATTER_MILLIS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS z").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_SECONDS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss ").withZoneUTC();
+    private final static DateTimeFormatter TO_STRING_FORMATTER_MILLIS = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss.SSS ").withZoneUTC();
     private final static DateTimeFormatter TO_STRING_FORMATTER_CUSTOM = DateTimeFormat.forPattern("yyyy-MM-dd HH:mm:ss").withZoneUTC();
-    private static final Pattern FROM_STRING_PATTERN = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})(?:\\.(\\d{1,9}))? UTC");
+    private static final Pattern FROM_STRING_PATTERN = Pattern.compile("(\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2})(?:\\.(\\d{1,9}))? (?:UTC|\\+?00\\:?00)");
 
     private final long seconds;
     private final int nano;
@@ -107,10 +107,10 @@ public class Timestamp
     public String toString()
     {
         if (nano == 0) {
-            return TO_STRING_FORMATTER_SECONDS.print(getEpochSecond() * 1000);
+            return TO_STRING_FORMATTER_SECONDS.print(getEpochSecond() * 1000) + "UTC";
 
         } else if (nano % 1000000 == 0) {
-            return TO_STRING_FORMATTER_MILLIS.print(toEpochMilli());
+            return TO_STRING_FORMATTER_MILLIS.print(toEpochMilli()) + "UTC";
 
         } else {
             StringBuffer sb = new StringBuffer();

data/embulk-core/src/test/java/org/embulk/spi/util/TestLineDecoder.java

@@ -4,6 +4,7 @@ import java.util.List;
 import java.util.ArrayList;
 import java.nio.ByteBuffer;
 import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
 import java.nio.charset.UnsupportedCharsetException;
 import com.google.common.collect.ImmutableList;
 import org.junit.Rule;
@@ -26,7 +27,7 @@ public class TestLineDecoder
     {
         ConfigSource config = Exec.newConfigSource();
         LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
-        assertEquals(Charset.forName("utf-8"), task.getCharset());
+        assertEquals(StandardCharsets.UTF_8, task.getCharset());
         assertEquals(Newline.CRLF, task.getNewline());
     }
 
@@ -37,22 +38,22 @@ public class TestLineDecoder
             .set("charset", "utf-16")
             .set("newline", "CRLF");
         LineDecoder.DecoderTask task = config.loadConfig(LineDecoder.DecoderTask.class);
-        assertEquals(Charset.forName("utf-16"), task.getCharset());
+        assertEquals(StandardCharsets.UTF_16, task.getCharset());
         assertEquals(Newline.CRLF, task.getNewline());
     }
 
-    private static LineDecoder.DecoderTask getExampleConfig()
+    private static LineDecoder.DecoderTask getExampleConfig(Charset charset, Newline newline)
     {
         ConfigSource config = Exec.newConfigSource()
-            .set("charset", "utf-8")
-            .set("newline", "LF");
+            .set("charset", charset)
+            .set("newline", newline);
         return config.loadConfig(LineDecoder.DecoderTask.class);
     }
 
     private static LineDecoder newDecoder(Charset charset, Newline newline, List<Buffer> buffers)
     {
         ListFileInput input = new ListFileInput(ImmutableList.of(buffers));
-        return new LineDecoder(input, getExampleConfig());
+        return new LineDecoder(input, getExampleConfig(charset, newline));
     }
 
     private static List<String> doDecode(Charset charset, Newline newline, List<Buffer> buffers)
@@ -70,10 +71,8 @@ public class TestLineDecoder
         return builder.build();
     }
 
-    private static List<Buffer> bufferList(String charsetName, String... sources) throws UnsupportedCharsetException
+    private static List<Buffer> bufferList(Charset charset, String... sources) throws UnsupportedCharsetException
     {
-        Charset charset = Charset.forName(charsetName);
-
         List<Buffer> buffers = new ArrayList<Buffer>();
         for (String source : sources) {
             ByteBuffer buffer = charset.encode(source);
@@ -87,8 +86,8 @@ public class TestLineDecoder
     public void testDecodeBasicAscii() throws Exception
     {
         List<String> decoded = doDecode(
-                Charset.forName("utf-8"), Newline.LF,
-                bufferList("utf-8", "test1\ntest2\ntest3\n"));
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "test1\ntest2\ntest3\n"));
         assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
     }
 
@@ -96,8 +95,8 @@ public class TestLineDecoder
     public void testDecodeBasicAsciiCRLF() throws Exception
     {
         List<String> decoded = doDecode(
-                Charset.forName("utf-8"), Newline.CRLF,
-                bufferList("utf-8", "test1\r\ntest2\r\ntest3\r\n"));
+                StandardCharsets.UTF_8, Newline.CRLF,
+                bufferList(StandardCharsets.UTF_8, "test1\r\ntest2\r\ntest3\r\n"));
         assertEquals(ImmutableList.of("test1", "test2", "test3"), decoded);
     }
 
@@ -105,8 +104,8 @@ public class TestLineDecoder
     public void testDecodeBasicAsciiTail() throws Exception
     {
         List<String> decoded = doDecode(
-                Charset.forName("utf-8"), Newline.LF,
-                bufferList("utf-8", "test1"));
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "test1"));
         assertEquals(ImmutableList.of("test1"), decoded);
     }
 
@@ -114,8 +113,8 @@ public class TestLineDecoder
     public void testDecodeChunksLF() throws Exception
     {
         List<String> decoded = doDecode(
-                Charset.forName("utf-8"), Newline.LF,
-                bufferList("utf-8", "t", "1", "\n", "t", "2"));
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "t", "1", "\n", "t", "2"));
         assertEquals(ImmutableList.of("t1", "t2"), decoded);
     }
 
@@ -123,10 +122,116 @@ public class TestLineDecoder
     public void testDecodeChunksCRLF() throws Exception
     {
         List<String> decoded = doDecode(
-                Charset.forName("utf-8"), Newline.CRLF,
-                bufferList("utf-8", "t", "1", "\r\n", "t", "2", "\r", "\n", "t3"));
+                StandardCharsets.UTF_8, Newline.CRLF,
+                bufferList(StandardCharsets.UTF_8, "t", "1", "\r\n", "t", "2", "\r", "\n", "t3"));
         assertEquals(ImmutableList.of("t1", "t2", "t3"), decoded);
     }
 
-    // TODO test multibytes
+    @Test
+    public void testDecodeBasicUTF8() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "てすと1\nテスト2\nてすと3\n"));
+        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicUTF8Tail() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "てすと1"));
+        assertEquals(ImmutableList.of("てすと1"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksUTF8LF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_8, Newline.LF,
+                bufferList(StandardCharsets.UTF_8, "て", "1", "\n", "す", "2"));
+        assertEquals(ImmutableList.of("て1", "す2"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksUTF8CRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_8, Newline.CRLF,
+                bufferList(StandardCharsets.UTF_8, "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
+        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicUTF16LE() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_16LE, Newline.LF,
+                bufferList(StandardCharsets.UTF_16LE, "てすと1\nテスト2\nてすと3\n"));
+        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicUTF16LETail() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_16LE, Newline.LF,
+                bufferList(StandardCharsets.UTF_16LE, "てすと1"));
+        assertEquals(ImmutableList.of("てすと1"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksUTF16LELF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_16LE, Newline.LF,
+                bufferList(StandardCharsets.UTF_16LE, "て", "1", "\n", "す", "2"));
+        assertEquals(ImmutableList.of("て1", "す2"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksUTF16LECRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                StandardCharsets.UTF_16LE, Newline.CRLF,
+                bufferList(StandardCharsets.UTF_16LE, "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
+        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicMS932() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("ms932"), Newline.LF,
+                bufferList(Charset.forName("ms932"), "てすと1\nテスト2\nてすと3\n"));
+        assertEquals(ImmutableList.of("てすと1", "テスト2", "てすと3"), decoded);
+    }
+
+    @Test
+    public void testDecodeBasicMS932Tail() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("ms932"), Newline.LF,
+                bufferList(Charset.forName("ms932"), "てすと1"));
+        assertEquals(ImmutableList.of("てすと1"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksMS932LF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("ms932"), Newline.LF,
+                bufferList(Charset.forName("ms932"), "て", "1", "\n", "す", "2"));
+        assertEquals(ImmutableList.of("て1", "す2"), decoded);
+    }
+
+    @Test
+    public void testDecodeChunksMS932CRLF() throws Exception
+    {
+        List<String> decoded = doDecode(
+                Charset.forName("ms932"), Newline.CRLF,
+                bufferList(Charset.forName("ms932"), "て", "1", "\r\n", "す", "2", "\r", "\n", "と3"));
+        assertEquals(ImmutableList.of("て1", "す2", "と3"), decoded);
+    }
 }

data/embulk-docs/src/index.rst

@@ -55,10 +55,14 @@ Documents
 
   * `File parser plugins <http://www.embulk.org/plugins/#file-parser>`_
 
+  * `File decoder plugins <http://www.embulk.org/plugins/#file-decoder>`_
+
   * `File formatter plugins <http://www.embulk.org/plugins/#file-formatter>`_
 
   * `Filter plugins <http://www.embulk.org/plugins/#filter>`_
 
+  * `File encoder plugins <http://www.embulk.org/plugins/#file-encoder>`_
+
   * `Executor plugins <http://www.embulk.org/plugins/#executor>`_
 
 .. toctree::

data/embulk-docs/src/recipe/scheduled-csv-load-to-elasticsearch-kibana4.rst

@@ -126,6 +126,13 @@ The generated config-complete.yml file should include complete information as fo
       nodes:
       - {host: localhost}
 
+Note: If the CSV file contains timestamp in local time zone, set default_timezone parameter to parser config as following (since time zone is assumed as UTC by default).
+
+.. code-block:: yaml
+
+      parser:
+        default_timezone: 'Asia/Tokyo'
+
 Now, you can run the bulk loading:
 
 .. code-block:: console

data/embulk-docs/src/release.rst

@@ -4,11 +4,13 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.7.5
     release/release-0.7.4
     release/release-0.7.3
     release/release-0.7.2
     release/release-0.7.1
     release/release-0.7.0
+    release/release-0.6.27
     release/release-0.6.26
     release/release-0.6.25
     release/release-0.6.24

data/embulk-docs/src/release/release-0.6.27.rst

@@ -0,0 +1,11 @@
+Release 0.6.27
+==================================
+
+Built-in plugins
+------------------
+
+* Backported fix of ``CsvTokenizer.InvalidValueException`` error handling problem in ``parser-csv`` plugin
+
+Release Date
+------------------
+2015-09-30

data/embulk-docs/src/release/release-0.7.5.rst

@@ -0,0 +1,22 @@
+Release 0.7.5
+==================================
+
+General Changes
+------------------
+
+* ``guess`` uses guess plugins configured at ``guess_plugins: [string]`` in ``exec:`` section in addition to ``-g, --guess NAMES`` command line argument.
+* ``guess`` doesn't create unnecessary ``exec:`` section.
+* Fixed timestamp deserialization error happens when a bulkload fails and cleanup runs.
+* Fixed permissions of files generated by ``new`` command.
+
+
+Built-in plugins
+------------------
+
+* ``guess-csv``: Guesses header line as much as possible when all column types are string or boolean
+* ``parser-csv``: Fixed line number shown in 'Skipped line' warning message
+
+
+Release Date
+------------------
+2015-10-01

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -229,8 +229,7 @@ public class CsvParserPlugin
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
         final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
-        LineDecoder lineDecoder = new LineDecoder(input, task);
-        final CsvTokenizer tokenizer = new CsvTokenizer(lineDecoder, task);
+        final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task);
         final String nullStringOrNull = task.getNullString().orNull();
         final boolean allowOptionalColumns = task.getAllowOptionalColumns();
         final boolean allowExtraColumns = task.getAllowExtraColumns();
@@ -241,7 +240,7 @@ public class CsvParserPlugin
             while (tokenizer.nextFile()) {
                 // skip the header lines for each file
                 for (; skipHeaderLines > 0; skipHeaderLines--) {
-                    if (lineDecoder.poll() == null) {
+                    if (!tokenizer.skipHeaderLine()) {
                         break;
                     }
                 }
@@ -356,8 +355,8 @@ public class CsvParserPlugin
                         pageBuilder.addRecord();
 
                     } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
-                        long lineNumber = tokenizer.getCurrentLineNumber();
                         String skippedLine = tokenizer.skipCurrentLine();
+                        long lineNumber = tokenizer.getCurrentLineNumber();
                         if (stopOnInvalidRecord) {
                             throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
                         }

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -56,8 +56,16 @@ public class CsvTokenizer
 
     public long getCurrentLineNumber()
     {
-        // returns actual line number. Internally, lineNumber starts at 0.
-        return lineNumber + 1;
+        return lineNumber;
+    }
+
+    public boolean skipHeaderLine()
+    {
+        boolean skipped = input.poll() != null;
+        if (skipped) {
+            lineNumber++;
+        }
+        return skipped;
     }
 
     // returns skipped line
@@ -70,10 +78,11 @@ public class CsvTokenizer
             // recover lines of quoted value
             skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
             unreadLines.addAll(quotedValueLines);
+            lineNumber -= quotedValueLines.size();
             if (line != null) {
                 unreadLines.add(line);
+                lineNumber -= 1;
             }
-            lineNumber -= quotedValueLines.size();
             quotedValueLines.clear();
         }
         recordState = RecordState.END;
@@ -305,7 +314,7 @@ public class CsvTokenizer
                         // column has trailing spaces and quoted. TODO should this be rejected?
 
                     } else {
-                        throw new InvalidValueException(String.format("Unexpected extra character (%c) after quoted value in %s", c, line));
+                        throw new InvalidValueException(String.format("Unexpected extra character '%c' after quoted value in '%s'", c, line));
                     }
                     break;
 

data/lib/embulk.rb

@@ -1,14 +1,4 @@
 module Embulk
-  if RUBY_PLATFORM =~ /java/i
-    def self.java?
-      true
-    end
-  else
-    def self.java?
-      false
-    end
-  end
-
   # logger should be setup first
   require 'embulk/logger'
 
@@ -29,19 +19,15 @@ module Embulk
 
   def self.require_classpath
     if __FILE__.include?("!")
-      # single jar
-      jar, resource = __FILE__.split("!", 2)
-      begin
-        require File.expand_path(jar)
-      rescue LoadError
-        # TODO fails if jar doesn't end with ".rb" or ".jar" but ignorable
-      end
+      # single jar. __FILE__ should point path/to/embulk.jar!/embulk.rb
+      # which means that embulk.jar is already loaded in this JVM.
 
     elsif __FILE__ =~ /^classpath:/
       # already in classpath
 
     else
-      # gem package
+      # gem package. __FILE__ should point path/to/embulk/lib/embulk.rb
+      # that requires here to load ../classpath/*.jar to start EmbulkEmbed.
       gem_root = File.expand_path('..', File.dirname(__FILE__))
       classpath_dir = File.join(gem_root, "classpath")
       jars = Dir.entries(classpath_dir).select{|f| f =~ /\.jar$/ }.sort

data/lib/embulk/command/embulk_new_plugin.rb

@@ -26,9 +26,9 @@ module Embulk
       #
       # Generate gemspec
       #
-      author = `git config user.name`.strip
+      author = `git config user.name`.strip rescue ""
       author = "YOUR_NAME" if author.empty?
-      email = `git config user.email`.strip
+      email = `git config user.email`.strip rescue ""
       email = "YOUR_NAME" if email.empty?
 
       # variables used in erb templates

data/lib/embulk/command/embulk_run.rb

@@ -130,7 +130,7 @@ module Embulk
         elsif File.exists?('Gemfile')
           bundle_path = '.'
         else
-          system_exit "'#{path}' already exists. You already ran 'embulk bundle new'. Please remove it, or run \"cd #{path}\" and \"embulk bundle\" instead"
+          system_exit "'#{gemfile_path}' already exists. You already ran 'embulk bundle new'. Please remove it, or run \"cd #{gemfile_path}\" and \"embulk bundle\" instead"
         end
 
         run_bundler(argv)
@@ -262,12 +262,14 @@ examples:
     else
       require 'json'
 
-      Embulk.setup(options.delete(:system_config))
-
       setup_plugin_paths(plugin_paths)
       setup_load_paths(load_paths)
       setup_classpaths(classpaths)
 
+      # call setup after setup_classpaths to allow users to overwrite
+      # embulk classes
+      Embulk.setup(options.delete(:system_config))
+
       begin
         case subcmd
         when :guess

data/lib/embulk/command/embulk_selfupdate.rb

@@ -29,8 +29,9 @@ module Embulk
       puts "Checking the latest version..."
       target_version = check_latest_version
 
-      if Gem::Version.new(target_version) <= Gem::Version.new(Embulk::VERSION)
-        puts "Already up-to-date. #{target_version} is the latest version."
+      current_version = Gem::Version.new(Embulk::VERSION)
+      if Gem::Version.new(target_version) <= current_version
+        puts "Already up-to-date. #{current_version} is the latest version."
         return
       end
 

data/lib/embulk/data/new/gitignore.erb

@@ -7,6 +7,10 @@
 /classpath/
 build/
 .idea
+/.settings/
+/.metadata/
+.classpath
+.project
 %elsif language == :ruby
 /.bundle/
 /Gemfile.lock

data/lib/embulk/data/new/java/decoder.java.erb

@@ -26,12 +26,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/new/java/encoder.java.erb

@@ -26,12 +26,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/new/java/file_input.java.erb

@@ -29,12 +29,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/new/java/file_output.java.erb

@@ -24,12 +24,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/new/java/filter.java.erb

@@ -23,12 +23,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
     }

data/lib/embulk/data/new/java/formatter.java.erb

@@ -23,12 +23,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
     }

data/lib/embulk/data/new/java/input.java.erb

@@ -26,12 +26,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/new/java/output.java.erb

@@ -26,12 +26,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
     }

data/lib/embulk/data/new/java/parser.java.erb

@@ -24,12 +24,12 @@ public class <%= java_class_name %>
         public int getOption1();
 
         // configuration option 2 (optional string, null is not allowed)
-        @Config("optoin2")
+        @Config("option2")
         @ConfigDefault("\"myvalue\"")
         public String getOption2();
 
         // configuration option 3 (optional string, null is allowed)
-        @Config("optoin3")
+        @Config("option3")
         @ConfigDefault("null")
         public Optional<String> getOption3();
 

data/lib/embulk/data/package_data.rb

@@ -26,13 +26,17 @@ module Embulk
     end
 
     def cp(src, dest_name)
-      path = dest_path_message(dest_name)
-      FileUtils.cp path(src), path
+      dest = dest_path_message(dest_name)
+      File.open(dest, "wb") do |dst_io|
+        File.open(path(src), "rb") do |src_io|
+          FileUtils.copy_stream src_io, dst_io
+        end
+      end
     end
 
     def cp_erb(src, dest_name)
-      path = dest_path_message(dest_name)
-      File.open(path, "w") {|f| f.write erb(src) }
+      dest = dest_path_message(dest_name)
+      File.open(dest, "wb") {|f| f.write erb(src) }
     end
 
     def dest_path(dest_name)
@@ -47,7 +51,8 @@ module Embulk
     end
 
     def set_executable(dest_name)
-      File.chmod(0755, dest_path(dest_name))
+      dest = dest_path(dest_name)
+      File.chmod(File.stat(dest).mode | 0111, dest)
     end
   end
 

data/lib/embulk/guess/csv.rb

@@ -123,7 +123,7 @@ module Embulk
           end
         end
 
-        header_line = (first_types != other_types && first_types.all? {|t| ["string", "boolean"].include?(t) })
+        header_line = (first_types != other_types && first_types.all? {|t| ["string", "boolean"].include?(t) }) || guess_string_header_line(sample_records)
 
         if header_line
           parser_guessed["skip_header_lines"] = skip_header_lines + 1
@@ -305,6 +305,22 @@ module Embulk
         end
       end
 
+      def guess_string_header_line(sample_records)
+        first = sample_records.first
+        first.count.times do |column_index|
+          lengths = sample_records.map {|row| row[column_index] }.compact.map {|v| v.to_s.size }
+          if lengths.size > 2
+            if array_variance(lengths[1..-1]) <= 0.2
+              avg = array_avg(lengths[1..-1])
+              if avg == 0.0 ? lengths[0] > 1 : (avg - lengths[0]).abs / avg > 0.7
+                return true
+              end
+            end
+          end
+        end
+        return false
+      end
+
       def array_sum(array)
         array.inject(0) {|r,i| r += i }
       end

data/lib/embulk/logger.rb

@@ -12,7 +12,7 @@ module Embulk
           @logger = a
         elsif a.is_a?(::Logger)
           @logger = StandardLoggerAdapter.new(a)
-        elsif Embulk.java? && (org.slf4j.Logger rescue nil) && a.is_a?(org.slf4j.Logger)
+        elsif RUBY_PLATFORM =~ /java/i && (org.slf4j.Logger rescue nil) && a.is_a?(org.slf4j.Logger)
           @logger = Slf4jAdapter.new(a)
         else
           @logger = StandardLoggerAdapter.new(*args)
@@ -43,7 +43,7 @@ module Embulk
 
     def initialize(*args)
       super
-      if Embulk.java?
+      if RUBY_PLATFORM =~ /java/i
         self.formatter = lambda do |severity,datetime,progname,message|
           "#{datetime.strftime("%Y-%m-%d %H:%M:%S.%3N %z")} [#{severity}] (#{java.lang.Thread.currentThread.name}): #{message}\n"
         end

data/lib/embulk/runner.rb

@@ -117,7 +117,7 @@ module Embulk
         when /\.yml$/
           @embed.newConfigLoader.fromYamlString File.read(config)
         else
-          raise ConfigError, "Unsupported file extension. Supported file extensions are .yml and .yml.liquid: #{config}"
+          raise ConfigError.new("Unsupported file extension. Supported file extensions are .yml and .yml.liquid: #{config}")
         end
 
       when Hash, DataSource

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.7.4'
+  VERSION = '0.7.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.7.4
+  version: 0.7.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-30 00:00:00.000000000 Z
+date: 2015-10-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jruby-jars
@@ -121,8 +121,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.7.4.jar
-- classpath/embulk-standards-0.7.4.jar
+- classpath/embulk-core-0.7.5.jar
+- classpath/embulk-standards-0.7.5.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar
@@ -177,6 +177,7 @@ files:
 - embulk-core/src/main/java/org/embulk/config/UserDataException.java
 - embulk-core/src/main/java/org/embulk/config/UserDataExceptions.java
 - embulk-core/src/main/java/org/embulk/exec/BulkLoader.java
+- embulk-core/src/main/java/org/embulk/exec/ConfigurableGuessInputPlugin.java
 - embulk-core/src/main/java/org/embulk/exec/ExecModule.java
 - embulk-core/src/main/java/org/embulk/exec/ExecutionInterruptedException.java
 - embulk-core/src/main/java/org/embulk/exec/ExecutionResult.java
@@ -410,6 +411,7 @@ files:
 - embulk-docs/src/release/release-0.6.24.rst
 - embulk-docs/src/release/release-0.6.25.rst
 - embulk-docs/src/release/release-0.6.26.rst
+- embulk-docs/src/release/release-0.6.27.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
 - embulk-docs/src/release/release-0.6.5.rst
@@ -422,6 +424,7 @@ files:
 - embulk-docs/src/release/release-0.7.2.rst
 - embulk-docs/src/release/release-0.7.3.rst
 - embulk-docs/src/release/release-0.7.4.rst
+- embulk-docs/src/release/release-0.7.5.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java