embulk 0.6.8 → 0.6.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a9e8a1f4e6ada76c6388248ae5ac9a2cdc6876ea
-  data.tar.gz: b21c680b792319509aa443b4e01809af8d3dcb56
+  metadata.gz: 19a42567d7e841a769ba329935d942926a062f66
+  data.tar.gz: 0bac8733cc4f5028e62865c08193520fce7f9d79
 SHA512:
-  metadata.gz: 7990996b97d6ae238dd20c8f997af9cb68f2d557b94b3d27388b8ede4e675c1c0f044072310d76ce37c4e4625497510ddecc9dee594a783ac91875f398c0137b
-  data.tar.gz: c2421ec9edd3f3e404302699526b1e226af19755186962b3b27074e53aebbfdb45dabf9e832c0b5a285bb40aad3e5a7aac7000724d735f43cae009b53150e2c9
+  metadata.gz: b74b82f6d740d513cce0eadc365824a78737a2cd40bcac32082ff51f03259cd720140eb7463530b3ffe8d8a7d8601a4d91de889a891760888c4b8ddfe9b4bede
+  data.tar.gz: 5a77c5eba35a06074289aab700f1351ced771f1f181cf5831d4ccf747692444063be0c32133a8d5bbf5d30fe6e636db398f591898135b351e81aeba7b90bb3db

data/build.gradle

@@ -11,7 +11,7 @@ def release_projects = [project(":embulk-core"), project(":embulk-standards")]
 
 allprojects {
     group = 'org.embulk'
-    version = '0.6.8'
+    version = '0.6.9'
 
     ext {
         jrubyVersion = '1.7.19'

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -1,6 +1,7 @@
 package org.embulk.spi.util;
 
 import java.io.Writer;
+import java.io.BufferedWriter;
 import java.io.OutputStreamWriter;
 import java.io.IOException;
 import java.nio.charset.Charset;
@@ -47,7 +48,7 @@ public class LineEncoder
         this.newline = task.getNewline().getString();
         this.underlyingFileOutput = out;
         this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH);
-        this.writer = new OutputStreamWriter(outputStream, encoder);
+        this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32*1024);
     }
 
     public void addNewLine()

data/embulk-docs/src/built-in.rst

@@ -285,17 +285,39 @@ The ``csv`` formatter plugin formats records using CSV or TSV format.
 Options
 ~~~~~~~~~~~~~~~~~~
 
-+----------------+----------+-------------------------------------------------------------------------------------------------------+------------------------+
-| name           | type     | description                                                                                           | required?              |
-+================+==========+=======================================================================================================+========================+
-| delimiter      | string   | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character     | ``,`` by default       |
-+----------------+----------+-------------------------------------------------------------------------------------------------------+------------------------+
-| header\_line   | boolean  | If true, write the header line with column name at the first line                                     |                        |
-+----------------+----------+-------------------------------------------------------------------------------------------------------+------------------------+
-| newline        | enum     | Newline character (CRLF, LF or CR)                                                                    | ``CRLF`` by default    |
-+----------------+----------+-------------------------------------------------------------------------------------------------------+------------------------+
-| charset        | enum     | Character encoding (eg. ISO-8859-1, UTF-8)                                                            | ``UTF-8`` by default   |
-+----------------+----------+-------------------------------------------------------------------------------------------------------+------------------------+
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| name                 | type    | description                                                                                           | required?              |
++======================+=========+=======================================================================================================+========================+
+| delimiter            | string  | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"`` or any single-byte character     | ``,`` by default       |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| quote                | string  | The character surrounding a quoted value                                                              | ``\"`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| quote\_policy        | enum    | Policy for quote (ALL, MINIMAL, NONE) (see below)                                                     | ``MINIMAL`` by default |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| escape               | string  | Escape character to escape a quote character when quote\_policy is ALL or MINIMAL                     | ``\"`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| header\_line         | boolean | If true, write the header line with column name at the first line                                     | ``true`` by default    |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| null_string          | string  | Expression of NULL values                                                                             | empty by default       |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| newline              | enum    | Newline character (CRLF, LF or CR)                                                                    | ``CRLF`` by default    |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| newline\_in\_field   | enum    | Newline character in each field (CRLF, LF, CR)                                                        | ``LF`` by default      |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+| charset              | enum    | Character encoding (eg. ISO-8859-1, UTF-8)                                                            | ``UTF-8`` by default   |
++----------------------+---------+-------------------------------------------------------------------------------------------------------+------------------------+
+
+The ``quote_policy`` option is used to determine field type to quote.
+
++------------+--------------------------------------------------------------------------------------------------------+
+| name       | description                                                                                            |
++============+========================================================================================================+
+| ALL        | Quote all fields                                                                                       |
++------------+--------------------------------------------------------------------------------------------------------+
+| MINIMAL    | Only quote those fields which contain delimiter, quote or any of the characters in lineterminator      |
++------------+--------------------------------------------------------------------------------------------------------+
+| NONE       | Never quote fields. When the delimiter occurs in field, escape with escape char                        |
++------------+--------------------------------------------------------------------------------------------------------+
 
 Example
 ~~~~~~~~~~~~~~~~~~
@@ -306,9 +328,14 @@ Example
       ...
       formatter:
       - type: csv
-        delimiter: "\t"
-        newline: LF
+        delimiter: '\t'
+        newline: CRLF
+        newline_in_field: LF
         charset: UTF-8
+        quote_policy: MINIMAL
+        quote: '"'
+        escape: '\\'
+        null_string: '\\N'
 
 Gzip encoder plugin
 ------------------

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.6.9
     release/release-0.6.8
     release/release-0.6.7
     release/release-0.6.6

data/embulk-docs/src/release/release-0.6.9.rst

@@ -0,0 +1,24 @@
+Release 0.6.9
+==================================
+
+Built-in plugins
+------------------
+
+* ``formatter-csv`` supports ``quote``, ``quote_policy``, ``escape``, ``newline_in_field``, and ``null_string`` options (@sakama++)
+
+  * ``quote_policy`` controls how to quote values. It can be either of ``ALL`` (quote all values), ``MINIMAL`` (quote if a value includes delimiter or quote character), or ``NONE`` (never quotes).
+
+  * ``escape`` controls how to escape quote character in a quoted string. The default is ``"`` (``"`` will be ``""``). Some applications may set it to ``\`` (``"`` will be ``\"``)
+
+  * ``null_string`` controls how to write NULL values. The default is ``""`` (empty string). You can use any strings such as ``\N`` or ``#N/A``.
+
+* ``guess-csv`` guesses columns which contain only 0 and 1 in first 32KB as long type rather than boolean type.
+
+General Changes
+------------------
+
+* ``spi.util.LineEncoder`` uses buffered writer. This improves performance of ``formatter-csv`` upto 10%.
+
+Release Date
+------------------
+2015-05-14

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.standards;
 
+import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableBiMap;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.config.Config;
@@ -20,11 +21,31 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutput;
 import org.embulk.spi.util.LineEncoder;
 
+import org.embulk.spi.util.Newline;
 import java.util.Map;
 
 public class CsvFormatterPlugin
         implements FormatterPlugin
 {
+    public enum QuotePolicy
+    {
+        ALL("ALL"),
+        MINIMAL("MINIMAL"),
+        NONE("NONE");
+
+        private final String string;
+
+        private QuotePolicy(String string)
+        {
+            this.string = string;
+        }
+
+        public String getString()
+        {
+            return string;
+        }
+    }
+
     public interface PluginTask
             extends LineEncoder.EncoderTask, TimestampFormatter.FormatterTask
     {
@@ -34,7 +55,27 @@ public class CsvFormatterPlugin
 
         @Config("delimiter")
         @ConfigDefault("\",\"")
-        public String getDelimiterChar();
+        public char getDelimiterChar();
+
+        @Config("quote")
+        @ConfigDefault("\"\\\"\"")
+        public char getQuoteChar();
+
+        @Config("quote_policy")
+        @ConfigDefault("\"MINIMAL\"")
+        public QuotePolicy getQuotePolicy();
+
+        @Config("escape")
+        @ConfigDefault("null")
+        public Optional<Character> getEscapeChar();
+
+        @Config("null_string")
+        @ConfigDefault("\"\"")
+        public String getNullString();
+
+        @Config("newline_in_field")
+        @ConfigDefault("\"LF\"")
+        public Newline getNewlineInField();
     }
 
     @Override
@@ -66,18 +107,24 @@ public class CsvFormatterPlugin
         final LineEncoder encoder = new LineEncoder(output, task);
         final Map<Integer, TimestampFormatter> timestampFormatters =
                 newTimestampFormatters(task, schema);
-        final String delimiter = task.getDelimiterChar();
+        final char delimiter = task.getDelimiterChar();
+        final QuotePolicy quotePolicy = task.getQuotePolicy();
+        final char quote = task.getQuoteChar() != '\0' ? task.getQuoteChar() : '"';
+        final char escape = task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : '\"');
+        final String newlineInField = task.getNewlineInField().getString();
+        final String nullString = task.getNullString();
 
         // create a file
         encoder.nextFile();
 
         // write header
         if (task.getHeaderLine()) {
-            writeHeader(schema, encoder, delimiter);
+            writeHeader(schema, encoder, delimiter, quotePolicy, quote, escape, newlineInField, nullString);
         }
 
         return new PageOutput() {
             private final PageReader pageReader = new PageReader(schema);
+            private final String delimiterString = String.valueOf(delimiter);
 
             public void add(Page page)
             {
@@ -88,7 +135,9 @@ public class CsvFormatterPlugin
                         {
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
-                                encoder.addText(Boolean.toString(pageReader.getBoolean(column)));
+                                addValue(Boolean.toString(pageReader.getBoolean(column)));
+                            } else {
+                                addNullString();
                             }
                         }
 
@@ -96,7 +145,9 @@ public class CsvFormatterPlugin
                         {
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
-                                encoder.addText(Long.toString(pageReader.getLong(column)));
+                                addValue(Long.toString(pageReader.getLong(column)));
+                            } else {
+                                addNullString();
                             }
                         }
 
@@ -104,7 +155,9 @@ public class CsvFormatterPlugin
                         {
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
-                                encoder.addText(Double.toString(pageReader.getDouble(column)));
+                                addValue(Double.toString(pageReader.getDouble(column)));
+                            } else {
+                                addNullString();
                             }
                         }
 
@@ -112,8 +165,9 @@ public class CsvFormatterPlugin
                         {
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
-                                // TODO escape and quoting
-                                encoder.addText(pageReader.getString(column));
+                                addValue(pageReader.getString(column));
+                            } else {
+                                addNullString();
                             }
                         }
 
@@ -122,18 +176,29 @@ public class CsvFormatterPlugin
                             addDelimiter(column);
                             if (!pageReader.isNull(column)) {
                                 Timestamp value = pageReader.getTimestamp(column);
-                                encoder.addText(timestampFormatters.get(column.getIndex()).format(value));
+                                addValue(timestampFormatters.get(column.getIndex()).format(value));
+                            } else {
+                                addNullString();
                             }
                         }
 
                         private void addDelimiter(Column column)
                         {
                             if (column.getIndex() != 0) {
-                                encoder.addText(delimiter);
+                                encoder.addText(delimiterString);
                             }
                         }
-                    });
 
+                        private void addValue(String v)
+                        {
+                            encoder.addText(setEscapeAndQuoteValue(v, delimiter, quotePolicy, quote, escape, newlineInField, nullString));
+                        }
+
+                        private void addNullString()
+                        {
+                            encoder.addText(nullString);
+                        }
+                    });
                     encoder.addNewLine();
                 }
             }
@@ -150,14 +215,72 @@ public class CsvFormatterPlugin
         };
     }
 
-    private void writeHeader(Schema schema, LineEncoder encoder, String delimiter)
+    private void writeHeader(Schema schema, LineEncoder encoder, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
     {
+        String delimiterString = String.valueOf(delimiter);
         for (Column column : schema.getColumns()) {
             if (column.getIndex() != 0) {
-                encoder.addText(delimiter);
+                encoder.addText(delimiterString);
             }
-            encoder.addText(column.getName());
+            encoder.addText(setEscapeAndQuoteValue(column.getName(), delimiter, policy, quote, escape, newline, nullString));
         }
         encoder.addNewLine();
     }
+
+    private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
+    {
+        StringBuilder escapedValue = new StringBuilder();
+        char previousChar = ' ';
+
+        boolean isRequireQuote = (policy == QuotePolicy.ALL || policy == QuotePolicy.MINIMAL && v.equals(nullString)) ? true : false;
+
+        for (int i = 0; i < v.length(); i++) {
+            char c = v.charAt(i);
+
+            if (c == quote) {
+                escapedValue.append(escape);
+                escapedValue.append(c);
+                isRequireQuote = true;
+            } else if (c == '\r') {
+                if (policy == QuotePolicy.NONE) {
+                    escapedValue.append(escape);
+                }
+                escapedValue.append(newline);
+                isRequireQuote = true;
+            } else if (c == '\n') {
+                if (previousChar != '\r') {
+                    if (policy == QuotePolicy.NONE) {
+                        escapedValue.append(escape);
+                    }
+                    escapedValue.append(newline);
+                    isRequireQuote = true;
+                }
+            } else if (c == delimiter) {
+                if (policy == QuotePolicy.NONE) {
+                    escapedValue.append(escape);
+                }
+                escapedValue.append(c);
+                isRequireQuote = true;
+            } else {
+                escapedValue.append(c);
+            }
+            previousChar = c;
+        }
+
+        if (policy != QuotePolicy.NONE && isRequireQuote) {
+            return setQuoteValue(escapedValue.toString(), quote);
+        } else {
+            return escapedValue.toString();
+        }
+    }
+
+    private String setQuoteValue(String v, char quote)
+    {
+        StringBuilder sb = new StringBuilder();
+        sb.append(quote);
+        sb.append(v);
+        sb.append(quote);
+
+        return sb.toString();
+    }
 }

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java

@@ -0,0 +1,309 @@
+package org.embulk.standards;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.junit.Rule;
+import org.junit.Test;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import static org.junit.Assert.assertEquals;
+import java.nio.charset.Charset;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.Newline;
+
+public class TestCsvFormatterPlugin
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource();
+
+        CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
+        assertEquals(Charset.forName("utf-8"), task.getCharset());
+        assertEquals(Newline.CRLF, task.getNewline());
+        assertEquals(true, task.getHeaderLine());
+        assertEquals(',', task.getDelimiterChar());
+        assertEquals('\"', task.getQuoteChar());
+        assertEquals(CsvFormatterPlugin.QuotePolicy.MINIMAL, task.getQuotePolicy());
+        assertEquals(false, task.getEscapeChar().isPresent());
+        assertEquals("", task.getNullString());
+        assertEquals(Newline.LF, task.getNewlineInField());
+    }
+
+    @Test
+    public void checkLoadConfig()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("charset", "utf-16")
+                .set("newline", "LF")
+                .set("header_line", false)
+                .set("delimiter", "\t")
+                .set("quote", "\\")
+                .set("quote_policy", "ALL")
+                .set("escape", "\"")
+                .set("null_string", "\\N")
+                .set("newline_in_field", "CRLF");
+
+        CsvFormatterPlugin.PluginTask task = config.loadConfig(CsvFormatterPlugin.PluginTask.class);
+        assertEquals(Charset.forName("utf-16"), task.getCharset());
+        assertEquals(Newline.LF, task.getNewline());
+        assertEquals(false, task.getHeaderLine());
+        assertEquals('\t', task.getDelimiterChar());
+        assertEquals('\\', task.getQuoteChar());
+        assertEquals(CsvFormatterPlugin.QuotePolicy.ALL, task.getQuotePolicy());
+        assertEquals('\"', (char) task.getEscapeChar().get());
+        assertEquals("\\N", task.getNullString());
+        assertEquals(Newline.CRLF, task.getNewlineInField());
+    }
+
+    @Test
+    public void testQuoteValue()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setQuoteValue", String.class, char.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        assertEquals("\"ABCD\"", method.invoke(formatter, "ABCD", '"'));
+        assertEquals("\"\"", method.invoke(formatter, "", '"'));
+        assertEquals("'ABCD'", method.invoke(formatter, "ABCD", '\''));
+        assertEquals("''", method.invoke(formatter, "", '\''));
+    }
+
+    @Test
+    public void testEscapeQuote()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
+        String newline = Newline.LF.getString();
+
+        assertEquals("\"AB\\\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '\\', newline, ""));
+        assertEquals("\"AB\"\"CD\"", method.invoke(formatter, "AB\"CD", delimiter, policy, '"', '"', newline, ""));
+    }
+
+    @Test
+    public void testQuotePolicyAll()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        char quote = '"';
+        char escape = '"';
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.ALL;
+        String newline = Newline.LF.getString();
+        String nullString = "";
+
+        @SuppressWarnings("unchecked")
+        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "\"true\"", "actual", "true"),
+                ImmutableMap.of("expected", "\"false\"", "actual", "false"),
+                ImmutableMap.of("expected", "\"0\"", "actual", "0"),
+                ImmutableMap.of("expected", "\"1\"", "actual", "1"),
+                ImmutableMap.of("expected", "\"1234\"", "actual", "1234"),
+                ImmutableMap.of("expected", "\"-1234\"", "actual", "-1234"),
+                ImmutableMap.of("expected", "\"+1234\"", "actual", "+1234"),
+                ImmutableMap.of("expected", "\"0x4d2\"", "actual", "0x4d2"),
+                ImmutableMap.of("expected", "\"123L\"", "actual", "123L"),
+                ImmutableMap.of("expected", "\"3.141592\"", "actual", "3.141592"),
+                ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
+                ImmutableMap.of("expected", "\"ABC\"", "actual", "ABC"),
+                ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
+                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
+                ImmutableMap.of("expected", "\"\"", "actual", ""),
+                ImmutableMap.of("expected", "\"NULL\"", "actual", "NULL"),
+                ImmutableMap.of("expected", "\"2015-01-01 12:01:01\"", "actual", "2015-01-01 12:01:01"),
+                ImmutableMap.of("expected", "\"20150101\"", "actual", "20150101"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+    }
+
+    @Test
+    public void testQuotePolicyMinimal()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        char quote = '"';
+        char escape = '"';
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
+        String newline = Newline.LF.getString();
+        String nullString = "";
+
+        @SuppressWarnings("unchecked")
+        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "true", "actual", "true"),
+                ImmutableMap.of("expected", "false", "actual", "false"),
+                ImmutableMap.of("expected", "0", "actual", "0"),
+                ImmutableMap.of("expected", "1", "actual", "1"),
+                ImmutableMap.of("expected", "1234", "actual", "1234"),
+                ImmutableMap.of("expected", "-1234", "actual", "-1234"),
+                ImmutableMap.of("expected", "+1234", "actual", "+1234"),
+                ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
+                ImmutableMap.of("expected", "123L", "actual", "123L"),
+                ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
+                ImmutableMap.of("expected", "\"1,000\"", "actual", "1,000"),
+                ImmutableMap.of("expected", "ABC", "actual", "ABC"),
+                ImmutableMap.of("expected", "\"ABC\"\"DEF\"", "actual", "ABC\"DEF"),
+                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"),
+                ImmutableMap.of("expected", "\"\"", "actual", ""),
+                ImmutableMap.of("expected", "NULL", "actual", "NULL"),
+                ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
+                ImmutableMap.of("expected", "20150101", "actual", "20150101"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+    }
+
+    @Test
+    public void testQuotePolicyNone()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        char quote = '"';
+        char escape = '"';
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.NONE;
+        String newline = Newline.LF.getString();
+        String nullString = "";
+
+        @SuppressWarnings("unchecked")
+        ImmutableList<ImmutableMap<String, String>> testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "true", "actual", "true"),
+                ImmutableMap.of("expected", "false", "actual", "false"),
+                ImmutableMap.of("expected", "0", "actual", "0"),
+                ImmutableMap.of("expected", "1", "actual", "1"),
+                ImmutableMap.of("expected", "1234", "actual", "1234"),
+                ImmutableMap.of("expected", "-1234", "actual", "-1234"),
+                ImmutableMap.of("expected", "+1234", "actual", "+1234"),
+                ImmutableMap.of("expected", "0x4d2", "actual", "0x4d2"),
+                ImmutableMap.of("expected", "123L", "actual", "123L"),
+                ImmutableMap.of("expected", "3.141592", "actual", "3.141592"),
+                ImmutableMap.of("expected", "1\",000", "actual", "1,000"),
+                ImmutableMap.of("expected", "ABC", "actual", "ABC"),
+                ImmutableMap.of("expected", "ABC\"\"DEF", "actual", "ABC\"DEF"),
+                ImmutableMap.of("expected", "ABC\"\nDEF", "actual", "ABC\nDEF"),
+                ImmutableMap.of("expected", "", "actual", ""),
+                ImmutableMap.of("expected", "NULL", "actual", "NULL"),
+                ImmutableMap.of("expected", "2015-01-01 12:01:01", "actual", "2015-01-01 12:01:01"),
+                ImmutableMap.of("expected", "20150101", "actual", "20150101"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+    }
+
+    @Test
+    public void testNewlineInField()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        char quote = '"';
+        char escape = '"';
+        String newline;
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
+        String nullString = "";
+
+        ImmutableList<ImmutableMap<String, String>> testCases;
+
+        newline = Newline.LF.getString();
+        testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\r\nDEF"),
+                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\rDEF"),
+                ImmutableMap.of("expected", "\"ABC\nDEF\"", "actual", "ABC\nDEF"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+
+
+        newline = Newline.CRLF.getString();
+        testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\r\nDEF"),
+                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\rDEF"),
+                ImmutableMap.of("expected", "\"ABC\r\nDEF\"", "actual", "ABC\nDEF"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+
+
+        newline = Newline.CR.getString();
+        testCases = ImmutableList.of(
+                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\r\nDEF"),
+                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\rDEF"),
+                ImmutableMap.of("expected", "\"ABC\rDEF\"", "actual", "ABC\nDEF"));
+
+        for (ImmutableMap testCase : testCases) {
+            String expected = (String) testCase.get("expected");
+            String actual = (String) testCase.get("actual");
+            assertEquals(expected, method.invoke(formatter, actual, delimiter, policy, quote, escape, newline, nullString));
+        }
+    }
+
+    @Test
+    public void testNullString()
+            throws NoSuchMethodException, SecurityException, IllegalAccessException, IllegalArgumentException, InvocationTargetException
+    {
+        Method method = CsvFormatterPlugin.class.getDeclaredMethod("setEscapeAndQuoteValue", String.class, char.class,
+                CsvFormatterPlugin.QuotePolicy.class, char.class, char.class, String.class, String.class);
+        method.setAccessible(true);
+        CsvFormatterPlugin formatter = new CsvFormatterPlugin();
+
+        char delimiter = ',';
+        char quote = '"';
+        char escape = '"';
+        CsvFormatterPlugin.QuotePolicy policy = CsvFormatterPlugin.QuotePolicy.MINIMAL;
+        String newline = Newline.LF.getString();
+
+        assertEquals("\"\"", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
+        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, ""));
+        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
+        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, ""));
+
+        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
+        assertEquals("\"N/A\"", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.MINIMAL, quote, escape, newline, "N/A"));
+        assertEquals("", method.invoke(formatter, "", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
+        assertEquals("N/A", method.invoke(formatter, "N/A", delimiter, CsvFormatterPlugin.QuotePolicy.NONE, quote, escape, newline, "N/A"));
+    }
+}

data/lib/embulk/guess/schema_guess.rb

@@ -87,7 +87,6 @@ module Embulk::Guess
         yes Yes YES
         t T y Y
         on On ON
-        1
       ].map {|k| [k, true] }]
 
       # When matching to false string, then retrun 'true'
@@ -96,7 +95,6 @@ module Embulk::Guess
         no No NO
         f N n N
         off Off OFF
-        0
       ].map {|k| [k, true] }]
 
       TYPE_COALESCE = Hash[{

data/lib/embulk/version.rb

@@ -1,3 +1,3 @@
 module Embulk
-  VERSION = '0.6.8'
+  VERSION = '0.6.9'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk
 version: !ruby/object:Gem::Version
-  version: 0.6.8
+  version: 0.6.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-05-13 00:00:00.000000000 Z
+date: 2015-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -300,6 +300,7 @@ files:
 - embulk-docs/src/release/release-0.6.6.rst
 - embulk-docs/src/release/release-0.6.7.rst
 - embulk-docs/src/release/release-0.6.8.rst
+- embulk-docs/src/release/release-0.6.9.rst
 - embulk-standards/build.gradle
 - embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java
 - embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java
@@ -313,6 +314,7 @@ files:
 - embulk-standards/src/main/java/org/embulk/standards/StandardPluginModule.java
 - embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java
 - embulk-standards/src/main/resources/META-INF/services/org.embulk.spi.Extension
+- embulk-standards/src/test/java/org/embulk/standards/TestCsvFormatterPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvParserPlugin.java
 - embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java
 - embulk.gemspec
@@ -406,8 +408,8 @@ files:
 - classpath/bval-jsr303-0.5.jar
 - classpath/commons-beanutils-core-1.8.3.jar
 - classpath/commons-lang3-3.1.jar
-- classpath/embulk-core-0.6.8.jar
-- classpath/embulk-standards-0.6.8.jar
+- classpath/embulk-core-0.6.9.jar
+- classpath/embulk-standards-0.6.9.jar
 - classpath/guava-18.0.jar
 - classpath/guice-4.0.jar
 - classpath/guice-multibindings-4.0.jar