embulk 0.8.35 → 0.8.36

data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java

@@ -9,6 +9,7 @@ import org.embulk.spi.Column;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.type.TimestampType;
+import org.joda.time.DateTimeZone;
 
 public class PagePrinter
 {
@@ -17,14 +18,29 @@ public class PagePrinter
     private final ArrayList<String> record;
 
     // TODO: Update this constructor because |TimestampFormater.FormatterTask| is deprecated since v0.6.14.
+    @Deprecated
     public PagePrinter(Schema schema, TimestampFormatter.FormatterTask task)
+    {
+        this(schema, task.getTimeZone());
+        // NOTE: Its deprecation is not actually from ScriptingContainer, though.
+        // TODO: Notify users about deprecated calls through the notification reporter.
+        if (!deprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated constructor of org.embulk.spi.util.PagePrinter.");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/827");
+            // The |deprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            deprecationWarned = true;
+        }
+    }
+
+    public PagePrinter(final Schema schema, final DateTimeZone timezone)
     {
         this.schema = schema;
         this.timestampFormatters = new TimestampFormatter[schema.getColumnCount()];
         for (int i=0; i < timestampFormatters.length; i++) {
             if (schema.getColumnType(i) instanceof TimestampType) {
                 TimestampType type = (TimestampType) schema.getColumnType(i);
-                timestampFormatters[i] = new TimestampFormatter(type.getFormat(), task.getTimeZone());
+                timestampFormatters[i] = new TimestampFormatter(type.getFormat(), timezone);
             }
         }
 
@@ -105,4 +121,6 @@ public class PagePrinter
             string = reader.getJson(column).toString();
         }
     }
+
+    private static boolean deprecationWarned = false;
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java

@@ -44,8 +44,17 @@ public abstract class AbstractDynamicColumnSetter
 
     public abstract void set(Value value);
 
+    @Deprecated
     public void setRubyObject(IRubyObject rubyObject)
     {
+        if (!deprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.util.dynamic.AbstractDynamicColumnSetter#setRubyObject");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/799");
+            // The |deprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            deprecationWarned = true;
+        }
+
         if (rubyObject == null || rubyObject instanceof RubyNil) {
             setNull();
         } else if (rubyObject instanceof RubyBoolean) {
@@ -80,4 +89,6 @@ public abstract class AbstractDynamicColumnSetter
             set(RubyValueApi.toValue(rubyObject.getRuntime(), rubyObject));
         }
     }
+
+    private static boolean deprecationWarned = false;
 }

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/SkipColumnSetter.java

@@ -51,7 +51,18 @@ public class SkipColumnSetter
     public void set(Value v)
     { }
 
+    @Deprecated
     @Override
     public void setRubyObject(IRubyObject rubyObject)
-    { }
+    {
+        if (!deprecationWarned) {
+            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.util.dynamic.SkipColumnSetter#setRubyObject");
+            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/799");
+            // The |deprecationWarned| flag is used only for warning messages.
+            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
+            deprecationWarned = true;
+        }
+    }
+
+    private static boolean deprecationWarned = false;
 }

data/embulk-core/src/main/resources/embulk/parent_first_packages.properties

@@ -40,6 +40,7 @@ edu.umd.cs.findbugs.annotations
 io.airlift.slice
 io.netty.buffer
 io.netty.util
+java
 javax.annotation
 javax.inject
 javax.validation

data/embulk-docs/build.gradle

@@ -1,3 +1,11 @@
+// TODO: Remove this block once rubygems.lasagna.io is back, or jruby-gradle-jar-plugin is upgraded to 1.5.0.
+// See also: https://github.com/jruby-gradle/jruby-gradle-plugin/issues/297
+// jruby-gradle-jar-plugin is not upgraded yet because its 1.5.0 depends on Java 8.
+repositories {
+    mavenLocal()
+    maven { url 'http://rubygems-proxy.torquebox.org/releases' }
+}
+
 apply plugin: 'com.github.jruby-gradle.base'
 
 import com.github.jrubygradle.JRubyExec

data/embulk-docs/src/built-in.rst

@@ -191,41 +191,53 @@ The ``csv`` parser plugin parses CSV and TSV files.
 Options
 ~~~~~~~~
 
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| name                       | type     | description                                                                                                    |              required?    |
-+============================+==========+================================================================================================================+===========================+
-| delimiter                  | string   | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"``                                           | ``,`` by default          |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| quote                      | string   | The character surrounding a quoted value. Setting ``null`` disables quoting.                                   | ``"`` by default          |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| escape                     | string   | Escape character to escape a special character. Setting ``null`` disables escaping.                            | ``\\`` by default         |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| skip\_header\_lines        | integer  | Skip this number of lines first. Set 1 if the file has header line.                                            | ``0`` by default          |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| null\_string               | string   | If a value is this string, converts it to NULL. For example, set ``\N`` for CSV files created by mysqldump     |                           |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| trim\_if\_not\_quoted      | boolean  | If true, remove spaces of a value if the value is not surrounded by the quote character                        | ``false`` by default      |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| comment\_line\_marker      | string   | Skip a line if the line begins with this string                                                                | null by default           |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| allow\_optional\_columns   | boolean  | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns   | ``false`` by default      |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| allow\_extra\_columns      | boolean  | If true, ignore too many columns. Otherwise, skip the row in case of too many columns                          | ``false`` by default      |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default     |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default      |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default        |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| default\_date              | string   | Set date part if the format doesn’t include date part.                                                         | ``1970-01-01`` by default |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default       |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| charset                    | enum     | Character encoding (eg. ISO-8859-1, UTF-8)                                                                     | ``UTF-8`` by default      |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
-| columns                    | hash     | Columns (see below)                                                                                            | required                  |
-+----------------------------+----------+----------------------------------------------------------------------------------------------------------------+---------------------------+
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| name                       | type     | description                                                                                                    |                               required?    |
++============================+==========+================================================================================================================+============================================+
+| delimiter                  | string   | Delimiter character such as ``,`` for CSV, ``"\t"`` for TSV, ``"|"``                                           | ``,`` by default                           |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| quote                      | string   | The character surrounding a quoted value. Setting ``null`` disables quoting.                                   | ``"`` by default                           |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| escape                     | string   | Escape character to escape a special character. Setting ``null`` disables escaping.                            | ``\\`` by default                          |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| skip\_header\_lines        | integer  | Skip this number of lines first. Set 1 if the file has header line.                                            | ``0`` by default                           |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| null\_string               | string   | If a value is this string, converts it to NULL. For example, set ``\N`` for CSV files created by mysqldump     |                                            |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| trim\_if\_not\_quoted      | boolean  | If true, remove spaces of a value if the value is not surrounded by the quote character                        | ``false`` by default                       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| quotes\_in\_quoted\_fields | enum     | Specify how to deal with irregular unescaped quote characters in quoted fields                                 | ``ACCEPT_ONLY_RFC4180_ESCAPED`` by default |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| comment\_line\_marker      | string   | Skip a line if the line begins with this string                                                                | null by default                            |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| allow\_optional\_columns   | boolean  | If true, set null to insufficient columns. Otherwise, skip the row in case of insufficient number of columns   | ``false`` by default                       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| allow\_extra\_columns      | boolean  | If true, ignore too many columns. Otherwise, skip the row in case of too many columns                          | ``false`` by default                       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| max\_quoted\_size\_limit   | integer  | Maximum number of bytes of a quoted value. If a value exceeds the limit, the row will be skipped               | ``131072`` by default                      |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| stop\_on\_invalid\_record  | boolean  | Stop bulk load transaction if a file includes invalid record (such as invalid timestamp)                       | ``false`` by default                       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| default\_timezone          | string   | Time zone of timestamp columns if the value itself doesn't include time zone description (eg. Asia/Tokyo)      | ``UTC`` by default                         |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| default\_date              | string   | Set date part if the format doesn’t include date part.                                                         | ``1970-01-01`` by default                  |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| newline                    | enum     | Newline character (CRLF, LF or CR)                                                                             | ``CRLF`` by default                        |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| charset                    | enum     | Character encoding (eg. ISO-8859-1, UTF-8)                                                                     | ``UTF-8`` by default                       |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+| columns                    | hash     | Columns (see below)                                                                                            | required                                   |
++----------------------------+----------+----------------------------------------------------------------------------------------------------------------+--------------------------------------------+
+
+The ``quotes_in_quoted_fields`` option specifies how to deal with irregular non-escaped stray quote characters.
+
++------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
+| name                                                 | description                                                                                                                                         |
++======================================================+=====================================================================================================================================================+
+| ACCEPT_ONLY_RFC4180_ESCAPED                          | Default. Accept only specified and RFC 4180-style escaped quote characters.                                                                         |
++------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
+| ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS | Accept stray quotes as-is in the field. Instead, it behaves undefined if delimiters are in fields. ``"a"b"`` goes ``a"b``. ``"a""b"`` goes ``a"b``. |
++------------------------------------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------+
 
 The ``columns`` option declares the list of columns. This CSV parser plugin ignores the header line.
 

data/embulk-docs/src/index.rst

@@ -7,9 +7,17 @@ Embulk
 ==================================
 
 .. image:: _static/embulk-logo-v2/embulk-logo-v2-sq-tr-small.png
-   :width: 512px
+   :width: 128px
    :target: https://github.com/embulk/embulk
 
+Highlights
+-----------
+
+* Embulk's announcement mailing list (read-only) is ready. Please feel free to subscribe! Embulk core members post important updates such as **key releases**, **compatibility information**, and **feedback requests to users**.
+
+  * `Embulk-announce <https://groups.google.com/forum/#!forum/embulk-announce>`_
+
+
 What's Embulk?
 ------------------
 

data/embulk-docs/src/release.rst

@@ -4,6 +4,7 @@ Release Notes
 .. toctree::
     :maxdepth: 1
 
+    release/release-0.8.36
     release/release-0.8.35
     release/release-0.8.34
     release/release-0.8.33

data/embulk-docs/src/release/release-0.8.36.rst

@@ -0,0 +1,32 @@
+Release 0.8.36
+==================================
+
+General Changes
+----------------
+
+* Load dependency JAR files embedded in plugin JAR [#792]
+* Improve timestamp parsing in Ruby plugins [#812] [#814]
+* Notify Embulk-announce mailing list in CLI [#816]
+
+Bug Fixes
+----------
+
+* Use single-quotes to quote path strings in YAML for Windows [#805]
+* Truncate output file before overwriting it [#807]
+* Fix typo in FALSE_STRINGS and add their test [#810]
+
+Built-in plugins
+-----------------
+
+* Add new option ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS in CSV parser [#809]
+
+Deprecations
+-------------
+
+* Deprecate JRuby-dependent classes and methods [#800] [#803] [#825]
+* Warn explicitly for deprecated methods [#821] [#826]
+
+
+Release Date
+------------------
+2017-10-24

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -77,6 +77,10 @@ public class CsvParserPlugin
         @ConfigDefault("false")
         boolean getTrimIfNotQuoted();
 
+        @Config("quotes_in_quoted_fields")
+        @ConfigDefault("\"ACCEPT_ONLY_RFC4180_ESCAPED\"")
+        QuotesInQuotedFields getQuotesInQuotedFields();
+
         @Config("max_quoted_size_limit")
         @ConfigDefault("131072") //128kB
         long getMaxQuotedSizeLimit();
@@ -98,6 +102,24 @@ public class CsvParserPlugin
         boolean getStopOnInvalidRecord();
     }
 
+    public enum QuotesInQuotedFields
+    {
+        ACCEPT_ONLY_RFC4180_ESCAPED,
+        ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS,
+        ;
+
+        @JsonCreator
+        public static QuotesInQuotedFields ofString(final String string)
+        {
+            for (final QuotesInQuotedFields value : values()) {
+                if (string.equals(value.toString())) {
+                    return value;
+                }
+            }
+            throw new ConfigException("\"quotes_in_quoted_fields\" must be one of [ACCEPT_ONLY_RFC4180_ESCAPED, ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS].");
+        }
+    }
+
     public static class QuoteCharacter
     {
         private final char character;

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -8,6 +8,7 @@ import java.util.ArrayDeque;
 import org.embulk.spi.DataException;
 import org.embulk.spi.util.LineDecoder;
 import org.embulk.config.ConfigException;
+import org.embulk.standards.CsvParserPlugin.QuotesInQuotedFields;
 
 public class CsvTokenizer
 {
@@ -31,6 +32,7 @@ public class CsvTokenizer
     private final char escape;
     private final String newline;
     private final boolean trimIfNotQuoted;
+    private final QuotesInQuotedFields quotesInQuotedFields;
     private final long maxQuotedSizeLimit;
     private final String commentLineMarker;
     private final LineDecoder input;
@@ -62,6 +64,12 @@ public class CsvTokenizer
         escape = task.getEscapeChar().or(CsvParserPlugin.EscapeCharacter.noEscape()).getCharacter();
         newline = task.getNewline().getString();
         trimIfNotQuoted = task.getTrimIfNotQuoted();
+        quotesInQuotedFields = task.getQuotesInQuotedFields();
+        if (trimIfNotQuoted && quotesInQuotedFields != QuotesInQuotedFields.ACCEPT_ONLY_RFC4180_ESCAPED) {
+            // The combination makes some syntax very ambiguous such as:
+            //     val1,  \"\"val2\"\"  ,val3
+            throw new ConfigException("[quotes_in_quoted_fields != ACCEPT_ONLY_RFC4180_ESCAPED] is not allowed to specify with [trim_if_not_quoted = true]");
+        }
         maxQuotedSizeLimit = task.getMaxQuotedSizeLimit();
         commentLineMarker = task.getCommentLineMarker().orNull();
         nullStringOrNull = task.getNullString().orNull();
@@ -313,9 +321,23 @@ public class CsvTokenizer
 
                     } else if (isQuote(c)) {
                         char next = peekNextChar();
-                        if (isQuote(next)) { // escaped quote
+                        final char nextNext = peekNextNextChar();
+                        if (isQuote(next) &&
+                            (quotesInQuotedFields != QuotesInQuotedFields.ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS ||
+                             (!isDelimiter(nextNext) && !isEndOfLine(nextNext)))) {
+                            // Escaped by preceding it with another quote.
+                            // A quote just before a delimiter or an end of line is recognized as a functional quote,
+                            // not just as a non-escaped stray "quote character" included the field, even if
+                            // ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS is specified.
                             quotedValue.append(line.substring(valueStartPos, linePos));
                             valueStartPos = ++linePos;
+                        } else if (quotesInQuotedFields == QuotesInQuotedFields.ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS &&
+                            !(isDelimiter(next) || isEndOfLine(next))) {
+                            // A non-escaped stray "quote character" in the field is processed as a regular character
+                            // if ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS is specified,
+                            if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
+                                throw new QuotedSizeLimitExceededException("The size of the quoted value exceeds the limit size (" + maxQuotedSizeLimit + ")");
+                            }
                         } else {
                             quotedValue.append(line.substring(valueStartPos, linePos - 1));
                             columnState = ColumnState.AFTER_QUOTED_VALUE;
@@ -427,6 +449,17 @@ public class CsvTokenizer
         }
     }
 
+    private char peekNextNextChar()
+    {
+        Preconditions.checkState(line != null, "peekNextNextChar is called after end of file");
+
+        if (linePos + 1 >= line.length()) {
+            return END_OF_LINE;
+        } else {
+            return line.charAt(linePos + 1);
+        }
+    }
+
     private boolean isSpace(char c)
     {
         return c == ' ';

data/embulk-standards/src/main/java/org/embulk/standards/StdoutOutputPlugin.java

@@ -1,6 +1,8 @@
 package org.embulk.standards;
 
 import java.util.List;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigDiff;
@@ -14,13 +16,17 @@ import org.embulk.spi.OutputPlugin;
 import org.embulk.spi.TransactionalPageOutput;
 import org.embulk.spi.PageReader;
 import org.embulk.spi.util.PagePrinter;
+import org.joda.time.DateTimeZone;
 
 public class StdoutOutputPlugin
         implements OutputPlugin
 {
     public interface PluginTask
-            extends Task, TimestampFormatter.FormatterTask
+            extends Task
     {
+        @Config("timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getTimeZone();
     }
 
     @Override
@@ -54,7 +60,7 @@ public class StdoutOutputPlugin
 
         return new TransactionalPageOutput() {
             private final PageReader reader = new PageReader(schema);
-            private final PagePrinter printer = new PagePrinter(schema, task);
+            private final PagePrinter printer = new PagePrinter(schema, task.getTimeZone());
 
             public void add(Page page)
             {

data/embulk-standards/src/test/java/org/embulk/standards/TestCsvTokenizer.java

@@ -293,6 +293,82 @@ public class TestCsvTokenizer
                     "\"trailing\n3\"  ,\"trailing\n4\"  "));
     }
 
+
+    @Test
+    public void parseWithDefaultQuotesInQuotedFields() throws Exception
+    {
+        reloadPluginTask();
+        assertEquals(expectedRecords(
+                         2,
+                         "foo\"bar", "foofoo\"barbar",
+                         "baz\"\"qux", "bazbaz\"\"quxqux"),
+                     parse(
+                         task,
+                         "\"foo\"\"bar\",\"foofoo\"\"barbar\"",
+                         "\"baz\"\"\"\"qux\",\"bazbaz\"\"\"\"quxqux\""));
+    }
+
+    @Test
+    public void parseWithQuotesInQuotedFields_ACCEPT_ONLY_RFC4180_ESCAPED() throws Exception
+    {
+        config.set("quotes_in_quoted_fields", "ACCEPT_ONLY_RFC4180_ESCAPED");
+        reloadPluginTask();
+        assertEquals(expectedRecords(
+                         2,
+                         "foo\"bar", "foofoo\"barbar",
+                         "baz\"\"qux", "bazbaz\"\"quxqux"),
+                     parse(
+                         task,
+                         "\"foo\"\"bar\",\"foofoo\"\"barbar\"",
+                         "\"baz\"\"\"\"qux\",\"bazbaz\"\"\"\"quxqux\""));
+    }
+
+    @Test
+    public void throwWithDefaultQuotesInQuotedFields() throws Exception
+    {
+        reloadPluginTask();
+        try {
+            parse(task, "\"foo\"bar\",\"hoge\"fuga\"");
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof CsvTokenizer.InvalidValueException);
+            assertEquals("Unexpected extra character 'b' after a value quoted by '\"'", e.getMessage());
+            return;
+        }
+    }
+
+    @Test
+    public void throwWithQuotesInQuotedFields_ACCEPT_ONLY_RFC4180_ESCAPED() throws Exception
+    {
+        config.set("quotes_in_quoted_fields", "ACCEPT_ONLY_RFC4180_ESCAPED");
+        reloadPluginTask();
+        try {
+            parse(task, "\"foo\"bar\",\"hoge\"fuga\"");
+            fail();
+        } catch (Exception e) {
+            assertTrue(e instanceof CsvTokenizer.InvalidValueException);
+            assertEquals("Unexpected extra character 'b' after a value quoted by '\"'", e.getMessage());
+            return;
+        }
+    }
+
+    @Test
+    public void parseWithQuotesInQuotedFields_ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS() throws Exception
+    {
+        config.set("quotes_in_quoted_fields", "ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS");
+        reloadPluginTask();
+        assertEquals(expectedRecords(
+                         2,
+                         "foo\"bar", "foofoo\"barbar",
+                         "baz\"\"qux", "bazbaz\"\"quxqux",
+                         "\"embulk\"", "\"embul\"\"k\""),
+                     parse(
+                         task,
+                         "\"foo\"bar\",\"foofoo\"\"barbar\"",
+                         "\"baz\"\"\"qux\",\"bazbaz\"\"\"\"quxqux\"",
+                         "\"\"\"embulk\"\",\"\"embul\"\"\"k\"\""));
+    }
+
     @Test
     public void throwQuotedSizeLimitExceededException() throws Exception
     {