embulk 0.6.10 → 0.6.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5f9bbe21037e6186e386b39176e66b8336ce8d20
-  data.tar.gz: 29fef3403d44ff6dc571d3e3c050efc23f77b770
+  metadata.gz: d45b70ffc7ebef537906a3f471d74f83d40f18ab
+  data.tar.gz: 792c390284e3b949d84b4b9860941620ba6d3f2a
 SHA512:
-  metadata.gz: 1481af06ce9c9aa2497ee25ee2c7b4f3be2578acf33e2be01ba06a0727cdc20523af16a4e7bae64658e286b2060e118e6eedd97cec085186843063f5246a6001
-  data.tar.gz: a3ab3e1c0e0f8477127dd30c5484210ae09845410ba0a747be942868061a002ad35e27e4deed4b63c6d49bca6477103728bb35378d12ae8363e40b24d8bea94b
+  metadata.gz: c297663fe08c2c1a83d68942facc2c00801fceab5acdf90a12dcef36b37eac272e10964ddf4f2dd6d869adab6cdeca9bc8925d763f51a6b27feac5c43b78ea65
+  data.tar.gz: ea1a1b4a7c3df3a6f61cb52f2487050d1f2b2a8abf1bf2616b9ee97b2f1d493e2a3b6135f1e32e3d5a0edf40d2a135615be309294e2bc7bdfd6a15c1808a8238

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.10'
+    version = '0.6.11'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-cli/src/main/sh/selfrun.sh

@@ -2,9 +2,103 @@
 : <<BAT
 @echo off
 
-java -jar %~f0 %*
+setlocal
+
+set this=%~f0
+set java_args=
+set jruby_args=
+set default_optimize=
+set overwrite_optimize=
+set status=
+set error=
+set args=
+
+rem In jar file, cannot goto ahread for some reason.
+
+for %%a in ( %* ) do (
+    call :check_arg %%a
+)
+
+if "%error%" == "true" exit /b 1
+
+set optimize=false
+if "%overwrite_optimize%" == "true" (
+    set optimize=true
+) else (
+    if "%default_optimize%" == "true" (
+        if not "%overwrite_optimize%" == "false" (
+            set optimize=true
+        )
+    )
+)
+
+if "%optimize%" == "true" (
+    set java_args=-XX:+AggressiveOpts -XX:+UseConcMarkSweepGC %java_args%
+) else (
+    set java_args=-XX:+AggressiveOpts -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -Xverify:none %java_args%
+)
+
+java %java_args% -jar %this% %jruby_args% %args%
+
+endlocal
+
+exit /b
+
+:check_arg
+set arg=%*
+
+rem Remove double quotations
+set p1=%arg:~0,1%
+set p1=%p1:"=%
+set p2=%arg:~-1,1%
+set p2=%p2:"=%
+set arg=%p1%%arg:~1,-1%%p2%
+
+if "%status%" == "rest" (
+    set args=%args% %arg%
+
+) else if "%status%" == "read" (
+    call :read_file %arg%
+
+) else if "%arg%" == "-J+O" (
+    set overwrite_optimize=true
+    set status=rest
+
+) else if "%arg%" == "-J-O" (
+    set overwrite_optimize=false
+    set status=rest
+
+) else if "%arg:~0,2%" == "-J" (
+    if not "%arg:~2%" == "" (
+        set java_args=%java_args% %arg:~2%
+    ) else (
+        set status=read
+    )
+
+) else if "%arg:~0,2%" == "-R" (
+    set jruby_args=%jruby_args% %arg:~2%
+
+) else if "%arg%" == "run" (
+    set default_optimize=true
+    set args=%args% %arg%
+    set status=rest
+
+) else (
+    set args=%args% %arg%
+    set status=rest
+)
+exit /b
+
+:read_file
+if not exist "%~1" (
+    echo "failed to load java argument file."
+    set error=true
+) else (
+    for /f "delims=" %%i in (%~1) do set java_args=%java_args% %%i
+)
+set status=
+exit /b
 
-exit /B
 BAT
 
 java_args=""

data/embulk-cli/src/test/java/org/embulk/cli/DummyMain.java

@@ -0,0 +1,23 @@
+package org.embulk.cli;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+
+public class DummyMain {
+
+    public static void main(String[] args) throws Exception {
+        System.out.println(Arrays.asList(args));
+        File thisFolder = new File(SelfrunTest.class.getResource("/org/embulk/cli/DummyMain.class").toURI()).getParentFile();
+        try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(new File(thisFolder, "args.txt")), Charset.defaultCharset()))) {
+            for (String arg : args) {
+                writer.write(arg);
+                writer.newLine();
+            }
+        }
+    }
+
+}

data/embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java

@@ -0,0 +1,281 @@
+package org.embulk.cli;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.nio.file.FileSystem;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.StandardOpenOption;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+public class SelfrunTest {
+
+    private static File testSelfrun;
+
+    @BeforeClass
+    public static void prepare() throws Exception {
+        File selfrun = findSelfrun();
+        FileSystem fs = FileSystems.getDefault();
+        String line = new String(Files.readAllBytes(fs.getPath(selfrun.getAbsolutePath())), Charset.defaultCharset());
+
+        File thisFolder = new File(SelfrunTest.class.getResource("/org/embulk/cli/SelfrunTest.class").toURI()).getParentFile();
+        testSelfrun = new File(thisFolder, System.getProperty("file.separator").equals("\\") ? "selfrun.bat" : "selfrun.sh");
+
+        File classpath = thisFolder.getParentFile().getParentFile().getParentFile();
+        line = line.replaceAll("java ", "java -classpath " + classpath.getAbsolutePath().replaceAll("\\\\", "\\\\\\\\") + " org.embulk.cli.DummyMain ");
+
+        // Modify selfrun so that arguments are written in 'args.txt' .
+        Files.write(fs.getPath(testSelfrun.getAbsolutePath()), line.getBytes(Charset.defaultCharset()), StandardOpenOption.CREATE);
+        if (!testSelfrun.setExecutable(true)) {
+        	throw new Exception("Cannot se executable.");
+        }
+    }
+
+
+    @Test
+    public void testNoArgument() throws Exception {
+        List<String> args = execute();
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath()),
+                args);
+    }
+
+    @Test
+    public void testArguments() throws Exception {
+        List<String> args = execute("a1", "a2", "\"a3=v3\"");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2",
+                "a3=v3"),
+                args);
+    }
+
+    @Test
+    public void testRun() throws Exception {
+        List<String> args = execute("run", "a1");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "run",
+                "a1"),
+                args);
+    }
+
+    @Test
+    public void testJpO() throws Exception {
+        List<String> args = execute("-J+O", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJmO() throws Exception {
+        List<String> args = execute("-J-O", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testR1() throws Exception {
+        List<String> args = execute("-Rr1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testR2() throws Exception {
+        List<String> args = execute("\"-Rr1=v1\"", "\"-Rr2=v2\"", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1=v1",
+                "r2=v2",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testRRun() throws Exception {
+        List<String> args = execute("-Rr1", "run", "a1");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+UseConcMarkSweepGC",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "run",
+                "a1"),
+                args);
+    }
+
+    @Test
+    public void testJ1() throws Exception {
+        List<String> args = execute("-J-Dj1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-Dj1",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJ2() throws Exception {
+        List<String> args = execute("\"-J-Dj1=v1\"", "\"-J-Dj2=v2\"", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "-Dj1=v1",
+                "-Dj2=v2",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJR() throws Exception {
+        List<String> args = execute("-Jj1", "-Rr1", "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "j1",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "r1",
+                "a1",
+                "a2"),
+                args);
+    }
+
+    @Test
+    public void testJFile() throws Exception {
+        File javaArgsFile = new File(testSelfrun.getParentFile(), "java_args.txt");
+        FileSystem fs = FileSystems.getDefault();
+        Files.write(fs.getPath(javaArgsFile.getAbsolutePath()), "j1 j2 j3".getBytes(Charset.defaultCharset()), StandardOpenOption.CREATE);
+
+        List<String> args = execute("-J", javaArgsFile.getAbsolutePath(), "a1", "a2");
+        assertEquals(Arrays.asList(
+                "-XX:+AggressiveOpts",
+                "-XX:+TieredCompilation",
+                "-XX:TieredStopAtLevel=1",
+                "-Xverify:none",
+                "j1",
+                "j2",
+                "j3",
+                "-jar",
+                testSelfrun.getAbsolutePath(),
+                "a1",
+                "a2"),
+                args);
+    }
+
+    private List<String> execute(String... arguments) throws Exception {
+        File temp = new File(testSelfrun.getParentFile(), "call-" + testSelfrun.getName());
+        try (BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(temp), Charset.defaultCharset()))) {
+            writer.write(testSelfrun.getAbsolutePath());
+            for (String argument : arguments) {
+                writer.write(" ");
+                writer.write(argument);
+            }
+        }
+        if (!temp.setExecutable(true)) {
+        	throw new Exception("Cannot se executable.");
+        }
+
+        File argsFile = new File(testSelfrun.getParentFile(), "args.txt");
+        if (argsFile.exists()) {
+            if (!argsFile.delete()) {
+            	throw new IOException("Cannot delete " + argsFile);
+            }
+        }
+
+        Process process = Runtime.getRuntime().exec(temp.getAbsolutePath());
+        int exitCode = process.waitFor();
+        if (exitCode != 0 || !argsFile.exists()) {
+            StringBuilder builder = new StringBuilder();
+            try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getErrorStream(), Charset.defaultCharset()))) {
+                builder.append(reader.readLine());
+                builder.append(System.getProperty("line.separator"));
+            }
+            throw new Exception(builder.toString());
+        }
+
+        FileSystem fs = FileSystems.getDefault();
+        List<String> args = Files.readAllLines(fs.getPath(argsFile.getAbsolutePath()), Charset.defaultCharset());
+        return args;
+    }
+
+    private static File findSelfrun() {
+        File folder = new File(".");
+        if (new File(folder, "embulk-cli").exists()) {
+            folder = new File(folder, "embulk-cli");
+        }
+        return new File(new File(new File(new File(folder, "src"), "main"), "sh"), "selfrun.sh");
+    }
+
+}

data/embulk-docs/src/built-in.rst

@@ -143,6 +143,8 @@ Options
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | trim\_if\_not\_quoted      | boolean  | If true, remove spaces of a value if the value is not surrounded by the quote character                        | ``false`` by default   |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
+| comment\_line\_marker      | string   | Skip a line if the line begins with this string                                                                | null by default        |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | allow\_optional\_columns   | boolean  | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns   | ``false`` by default   |
 +----------------------------+----------+----------------------------------------------------------------------------------------------------------------+------------------------+
 | allow\_extra\_columns      | boolean  | If true, ignore too many columns. Otherwise, skip the row in case of too many columns                          | ``false`` by default   |
@@ -204,6 +206,7 @@ Example
         escape: ''
         null_string: 'NULL'
         skip_header_lines: 1
+        comment_line_marker: '#'
         columns:
         - {name: id, type: long}
         - {name: account, type: long}

data/embulk-docs/src/release/release-0.6.11.rst

@@ -0,0 +1,19 @@
+Release 0.6.11
+==================================
+
+Built-in plugins
+------------------
+
+* ``input-file`` plugin sets ``last_path`` when there are no input files.
+* ``parser-csv`` supports **comment_line_marker** option to skip lines starting with comment characters such as '#'.
+* Fixed a bug where ``guess-csv`` guesses timestamp wrongly if order of date is month-day-year.
+
+General Changes
+------------------
+
+* Command line execution supports ``-J``, ``-R``, ``-J+O``, and ``-J-O`` arguments on Windows (@hito4++)
+
+
+Release Date
+------------------
+2015-05-30

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.11
     release/release-0.6.10
     release/release-0.6.9
     release/release-0.6.8

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -75,6 +75,10 @@ public class CsvParserPlugin
         @ConfigDefault("131072") //128kB
         public long getMaxQuotedSizeLimit();
 
+        @Config("comment_line_marker")
+        @ConfigDefault("null")
+        public Optional<String> getCommentLineMarker();
+
         @Config("allow_optional_columns")
         @ConfigDefault("false")
         public boolean getAllowOptionalColumns();

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -27,6 +27,7 @@ public class CsvTokenizer
     private final String newline;
     private final boolean trimIfNotQuoted;
     private final long maxQuotedSizeLimit;
+    private final String commentLineMarker;
     private final LineDecoder input;
 
     private RecordState recordState = RecordState.END;  // initial state is end of a record. nextRecord() must be called first
@@ -46,6 +47,7 @@ public class CsvTokenizer
         newline = task.getNewline().getString();
         trimIfNotQuoted = task.getTrimIfNotQuoted();
         maxQuotedSizeLimit = task.getMaxQuotedSizeLimit();
+        commentLineMarker = task.getCommentLineMarker().orNull();
         this.input = input;
     }
 
@@ -93,7 +95,7 @@ public class CsvTokenizer
         }
     }
 
-    private boolean nextLine(boolean ignoreEmptyLine)
+    private boolean nextLine(boolean skipEmptyLine)
     {
         while (true) {
             if (!unreadLines.isEmpty()) {
@@ -107,7 +109,10 @@ public class CsvTokenizer
             linePos = 0;
             lineNumber++;
 
-            if (!line.isEmpty() || !ignoreEmptyLine) {
+            boolean skip = skipEmptyLine && (
+                        line.isEmpty() ||
+                        (commentLineMarker != null && line.startsWith(commentLineMarker)));
+            if (!skip) {
                 return true;
             }
         }

data/embulk-standards/src/main/java/org/embulk/standards/LocalFileInputPlugin.java

@@ -79,10 +79,22 @@ public class LocalFileInputPlugin
 
         control.run(taskSource, taskCount);
 
-        List<String> files = new ArrayList<String>(task.getFiles());
-        Collections.sort(files);
-        return Exec.newConfigDiff().
-            set("last_path", files.get(files.size() - 1));
+        // build next config
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        // last_path
+        if (task.getFiles().isEmpty()) {
+            // keep the last value
+            if (task.getLastPath().isPresent()) {
+                configDiff.set("last_path", task.getLastPath().get());
+            }
+        } else {
+            List<String> files = new ArrayList<String>(task.getFiles());
+            Collections.sort(files);
+            configDiff.set("last_path", files.get(files.size() - 1));
+        }
+
+        return configDiff;
     }
 
     @Override

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java

@@ -226,6 +226,20 @@ public class TestCsvTokenizer
                     "\n\"a\\\"aa\",\"b,bb\\\"\"\n\n\"cc\"\"c\",\"\"\"ddd\"\n,\"\"\n")));
     }
 
+    @Test
+    public void testCommentLineMarker() throws Exception
+    {
+        config.set("comment_line_marker", JsonNodeFactory.instance.textNode("#"));
+        reloadPluginTask();
+        assertEquals(expectedRecords(2,
+                        "aaa", "bbb",
+                        "eee", "fff"),
+            parse(task,
+                    "aaa,bbb",
+                    "#ccc,ddd",
+                    "eee,fff"));
+    }
+
     @Test
     public void trimNonQuotedValues() throws Exception
     {

data/lib/embulk/data/new/README.md.erb

@@ -93,7 +93,7 @@ out:
 (If guess supported) you don't have to write `<%= category %>:` section in the configuration file. After writing `in:` section, you can let embulk guess `<%= category %>:` section using this command:
 
 ```
-$ embulk install <%= project_name %>
+$ embulk gem install <%= project_name %>
 $ embulk guess -g <%= name %> config.yml -o guessed.yml
 ```
 %end

data/lib/embulk/guess/csv.rb

@@ -24,6 +24,11 @@ module Embulk
         "\\N",  # MySQL LOAD, Hive STORED AS TEXTFILE
       ]
 
+      COMMENT_LINE_MARKER_CANDIDATES = [
+        "#",
+        "//",
+      ]
+
       MAX_SKIP_LINES = 10
       NO_SKIP_DETECT_LINES = 10
 
@@ -56,9 +61,12 @@ module Embulk
         end
 
         sample_records = split_lines(parser_guessed, sample_lines, delim)
+
         skip_header_lines = guess_skip_header_lines(sample_records)
         sample_records = sample_records[skip_header_lines..-1]
 
+        comment_line_marker, sample_records = guess_comment_line_marker(sample_records)
+
         first_types = SchemaGuess.types_from_array_records(sample_records[0, 1])
         other_types = SchemaGuess.types_from_array_records(sample_records[1..-1])
 
@@ -75,6 +83,8 @@ module Embulk
           parser_guessed["skip_header_lines"] = skip_header_lines
         end
 
+        parser_guessed["comment_line_marker"] = comment_line_marker  # always set comment_line_marker even if it's null
+
         parser_guessed["allow_extra_columns"] = false
         parser_guessed["allow_optional_columns"] = false
 
@@ -113,7 +123,10 @@ module Embulk
               columns = []
               while true
                 begin
-                  columns << tokenizer.nextColumn
+                  column = tokenizer.nextColumn
+                  quoted = tokenizer.wasQuotedColumn
+                  column.define_singleton_method(:quoted?) { quoted }
+                  columns << column
                 rescue org.embulk.standards.CsvTokenizer::TooFewColumnsException
                   rows << columns
                   break
@@ -200,21 +213,38 @@ module Embulk
           count = counts.inject(0) {|r,c| r + c }
           [str, count]
         end.select {|str,count| count > 0 }.sort_by {|str,count| -count }
-        found = guessed.first
-        return found ? found[0] : nil
+        found_str, found_count = guessed.first
+        return found_str ? found_str : nil
       end
 
       def guess_skip_header_lines(sample_records)
         counts = sample_records.map {|records| records.size }
         (1..[MAX_SKIP_LINES, counts.length - 1].min).each do |i|
           check_row_count = counts[i-1]
-          if counts[i, NO_SKIP_DETECT_LINES].all? {|c| c == check_row_count }
+          if counts[i, NO_SKIP_DETECT_LINES].all? {|c| c <= check_row_count }
             return i - 1
           end
         end
         return 0
       end
 
+      def guess_comment_line_marker(sample_records)
+        guessed = COMMENT_LINE_MARKER_CANDIDATES.map do |str|
+          regexp = /^#{Regexp.quote(str)}/
+          records = sample_records.reject do |records|
+            !records[0].quoted? && !NULL_STRING_CANDIDATES.include?(records[0]) && records[0] =~ regexp
+          end
+          count = sample_records.size - records.size
+          [str, count, records]
+        end.select {|str,count,records| count > 0 }.sort_by {|str,count,records| -count }
+        found_str, found_count, found_records = guessed.first
+        if found_str
+          return found_str, found_records
+        else
+          return nil, sample_records
+        end
+      end
+
       def array_sum(array)
         array.inject(0) {|r,i| r += i }
       end

data/lib/embulk/guess/time_format_guess.rb

@@ -196,12 +196,12 @@ module Embulk::Guess
 
           parts << :year
           part_options << nil
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :month
           part_options << part_heading_option(dm["month"])
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :day
           part_options << part_heading_option(dm["day"])
 
@@ -210,30 +210,28 @@ module Embulk::Guess
 
           parts << :month
           part_options << part_heading_option(dm["month"])
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :day
           part_options << part_heading_option(dm["day"])
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :year
           part_options << nil
-          delimiters << date_delim
 
         elsif dm = (/^#{DMY}(?<rest>.*?)$/.match(text) or /^#{DMY_NODELIM}(?<rest>.*?)$/.match(text))
           date_delim = dm["date_delim"] rescue ""
 
           parts << :day
           part_options << part_heading_option(dm["day"])
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :month
           part_options << part_heading_option(dm["month"])
-          delimiters << date_delim
 
+          delimiters << date_delim
           parts << :year
           part_options << nil
-          delimiters << date_delim
 
         else
           date_delim = ""

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.10'
+  VERSION = '0.6.11'
 end

data/test/guess/test_time_format_guess.rb

@@ -45,6 +45,13 @@ class TimeFormatGuessTest < ::Test::Unit::TestCase
     assert_guess "%m.%d.%Y", "01.01.2014"
     assert_guess "%d/%m/%Y", "13/01/2014"
     assert_guess "%d/%m/%Y", "21/01/2014"
+
+    assert_guess "%d/%m/%Y %H-%M-%S,%N",    "21/01/2014 01-01-01,000000001"
+    assert_guess "%d/%m/%Y %H-%M-%S,%N",    "21/01/2014 01-01-01,000001"
+    assert_guess "%d/%m/%Y %H-%M-%S,%L",    "21/01/2014 01-01-01,001"
+    assert_guess "%d/%m/%Y %H-%M-%S",       "21/01/2014 01-01-01"
+    assert_guess "%d/%m/%Y %H-%M",          "21/01/2014 01-01"
+    assert_guess "%d/%m/%Y",                "21/01/2014"
   end
 
   def test_format_borders

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.10
+  version: 0.6.11
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-22 00:00:00.000000000 Z
+date: 2015-05-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -100,6 +100,8 @@ files:
 - embulk-cli/build.gradle
 - embulk-cli/src/main/java/org/embulk/cli/Main.java
 - embulk-cli/src/main/sh/selfrun.sh
+- embulk-cli/src/test/java/org/embulk/cli/DummyMain.java
+- embulk-cli/src/test/java/org/embulk/cli/SelfrunTest.java
 - embulk-core/build.gradle
 - embulk-core/src/main/java/org/embulk/EmbulkService.java
 - embulk-core/src/main/java/org/embulk/command/PreviewPrinter.java
@@ -294,6 +296,7 @@ files:
 - embulk-docs/src/release/release-0.6.0.rst
 - embulk-docs/src/release/release-0.6.1.rst
 - embulk-docs/src/release/release-0.6.10.rst
+- embulk-docs/src/release/release-0.6.11.rst
 - embulk-docs/src/release/release-0.6.2.rst
 - embulk-docs/src/release/release-0.6.3.rst
 - embulk-docs/src/release/release-0.6.4.rst
@@ -409,8 +412,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.10.jar
-- classpath/embulk-standards-0.6.10.jar
+- classpath/embulk-core-0.6.11.jar
+- classpath/embulk-standards-0.6.11.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar