embulk 0.8.38-java → 0.10.27-java

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -1,416 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableSet;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonValue;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigException;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.time.TimestampParseException;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.json.JsonParseException;
-import org.embulk.spi.Column;
-import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.DataException;
-import org.embulk.spi.util.LineDecoder;
-import org.embulk.spi.util.Timestamps;
-import org.slf4j.Logger;
-
-public class CsvParserPlugin
-        implements ParserPlugin
-{
-    private static final ImmutableSet<String> TRUE_STRINGS =
-        ImmutableSet.of(
-                "true", "True", "TRUE",
-                "yes", "Yes", "YES",
-                "t", "T", "y", "Y",
-                "on", "On", "ON",
-                "1");
-
-    public interface PluginTask
-            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
-    {
-        @Config("columns")
-        SchemaConfig getSchemaConfig();
-
-        @Config("header_line")
-        @ConfigDefault("null")
-        Optional<Boolean> getHeaderLine();
-
-        @Config("skip_header_lines")
-        @ConfigDefault("0")
-        int getSkipHeaderLines();
-        void setSkipHeaderLines(int n);
-
-        @Config("delimiter")
-        @ConfigDefault("\",\"")
-        String getDelimiter();
-
-        @Config("quote")
-        @ConfigDefault("\"\\\"\"")
-        Optional<QuoteCharacter> getQuoteChar();
-
-        @Config("escape")
-        @ConfigDefault("\"\\\\\"")
-        Optional<EscapeCharacter> getEscapeChar();
-
-        // Null value handling: if the CsvParser found 'non-quoted empty string's,
-        // it replaces them to string that users specified like "\N", "NULL".
-        @Config("null_string")
-        @ConfigDefault("null")
-        Optional<String> getNullString();
-
-        @Config("trim_if_not_quoted")
-        @ConfigDefault("false")
-        boolean getTrimIfNotQuoted();
-
-        @Config("quotes_in_quoted_fields")
-        @ConfigDefault("\"ACCEPT_ONLY_RFC4180_ESCAPED\"")
-        QuotesInQuotedFields getQuotesInQuotedFields();
-
-        @Config("max_quoted_size_limit")
-        @ConfigDefault("131072") //128kB
-        long getMaxQuotedSizeLimit();
-
-        @Config("comment_line_marker")
-        @ConfigDefault("null")
-        Optional<String> getCommentLineMarker();
-
-        @Config("allow_optional_columns")
-        @ConfigDefault("false")
-        boolean getAllowOptionalColumns();
-
-        @Config("allow_extra_columns")
-        @ConfigDefault("false")
-        boolean getAllowExtraColumns();
-
-        @Config("stop_on_invalid_record")
-        @ConfigDefault("false")
-        boolean getStopOnInvalidRecord();
-    }
-
-    public enum QuotesInQuotedFields
-    {
-        ACCEPT_ONLY_RFC4180_ESCAPED,
-        ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS,
-        ;
-
-        @JsonCreator
-        public static QuotesInQuotedFields ofString(final String string)
-        {
-            for (final QuotesInQuotedFields value : values()) {
-                if (string.equals(value.toString())) {
-                    return value;
-                }
-            }
-            throw new ConfigException("\"quotes_in_quoted_fields\" must be one of [ACCEPT_ONLY_RFC4180_ESCAPED, ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS].");
-        }
-    }
-
-    public static class QuoteCharacter
-    {
-        private final char character;
-
-        public QuoteCharacter(char character)
-        {
-            this.character = character;
-        }
-
-        public static QuoteCharacter noQuote()
-        {
-            return new QuoteCharacter(CsvTokenizer.NO_QUOTE);
-        }
-
-        @JsonCreator
-        public static QuoteCharacter ofString(String str)
-        {
-            if (str.length() >= 2) {
-                throw new ConfigException("\"quote\" option accepts only 1 character.");
-            } else if (str.isEmpty()) {
-                Exec.getLogger(CsvParserPlugin.class).warn("Setting '' (empty string) to \"quote\" option is obsoleted. Currently it becomes '\"' automatically but this behavior will be removed. Please set '\"' explicitly.");
-                return new QuoteCharacter('"');
-            } else {
-                return new QuoteCharacter(str.charAt(0));
-            }
-        }
-
-        @JsonIgnore
-        public char getCharacter()
-        {
-            return character;
-        }
-
-        @JsonValue
-        public String getOptionalString()
-        {
-            return new String(new char[] { character });
-        }
-
-        @Override
-        public boolean equals(Object obj)
-        {
-            if (!(obj instanceof QuoteCharacter)) {
-                return false;
-            }
-            QuoteCharacter o = (QuoteCharacter) obj;
-            return character == o.character;
-        }
-    }
-
-    public static class EscapeCharacter
-    {
-        private final char character;
-
-        public EscapeCharacter(char character)
-        {
-            this.character = character;
-        }
-
-        public static EscapeCharacter noEscape()
-        {
-            return new EscapeCharacter(CsvTokenizer.NO_ESCAPE);
-        }
-
-        @JsonCreator
-        public static EscapeCharacter ofString(String str)
-        {
-            if (str.length() >= 2) {
-                throw new ConfigException("\"escape\" option accepts only 1 character.");
-            } else if (str.isEmpty()) {
-                Exec.getLogger(CsvParserPlugin.class).warn("Setting '' (empty string) to \"escape\" option is obsoleted. Currently it becomes null automatically but this behavior will be removed. Please set \"escape: null\" explicitly.");
-                return noEscape();
-            } else {
-                return new EscapeCharacter(str.charAt(0));
-            }
-        }
-
-        @JsonIgnore
-        public char getCharacter()
-        {
-            return character;
-        }
-
-        @JsonValue
-        public String getOptionalString()
-        {
-            return new String(new char[] { character });
-        }
-
-        @Override
-        public boolean equals(Object obj)
-        {
-            if (!(obj instanceof EscapeCharacter)) {
-                return false;
-            }
-            EscapeCharacter o = (EscapeCharacter) obj;
-            return character == o.character;
-        }
-    }
-
-    private final Logger log;
-
-    public CsvParserPlugin()
-    {
-        log = Exec.getLogger(CsvParserPlugin.class);
-    }
-
-    @Override
-    public void transaction(ConfigSource config, ParserPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // backward compatibility
-        if (task.getHeaderLine().isPresent()) {
-            if (task.getSkipHeaderLines() > 0) {
-                throw new ConfigException("'header_line' option is invalid if 'skip_header_lines' is set.");
-            }
-            if (task.getHeaderLine().get()) {
-                task.setSkipHeaderLines(1);
-            } else {
-                task.setSkipHeaderLines(0);
-            }
-        }
-
-        control.run(task.dump(), task.getSchemaConfig().toSchema());
-    }
-
-    @Override
-    public void run(TaskSource taskSource, final Schema schema,
-            FileInput input, PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
-        final JsonParser jsonParser = new JsonParser();
-        final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task);
-        final boolean allowOptionalColumns = task.getAllowOptionalColumns();
-        final boolean allowExtraColumns = task.getAllowExtraColumns();
-        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
-        final int skipHeaderLines = task.getSkipHeaderLines();
-
-        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-            while (tokenizer.nextFile()) {
-                // skip the header lines for each file
-                for (int skipHeaderLineNumber = skipHeaderLines; skipHeaderLineNumber > 0; skipHeaderLineNumber--) {
-                    if (!tokenizer.skipHeaderLine()) {
-                        break;
-                    }
-                }
-
-                if (!tokenizer.nextRecord()) {
-                    // empty file
-                    continue;
-                }
-
-                while (true) {
-                    boolean hasNextRecord;
-
-                    try {
-                        schema.visitColumns(new ColumnVisitor() {
-                            public void booleanColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    pageBuilder.setBoolean(column, TRUE_STRINGS.contains(v));
-                                }
-                            }
-
-                            public void longColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    try {
-                                        pageBuilder.setLong(column, Long.parseLong(v));
-                                    } catch (NumberFormatException e) {
-                                        // TODO support default value
-                                        throw new CsvRecordValidateException(e);
-                                    }
-                                }
-                            }
-
-                            public void doubleColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    try {
-                                        pageBuilder.setDouble(column, Double.parseDouble(v));
-                                    } catch (NumberFormatException e) {
-                                        // TODO support default value
-                                        throw new CsvRecordValidateException(e);
-                                    }
-                                }
-                            }
-
-                            public void stringColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    pageBuilder.setString(column, v);
-                                }
-                            }
-
-                            public void timestampColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    try {
-                                        pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(v));
-                                    } catch (TimestampParseException e) {
-                                        // TODO support default value
-                                        throw new CsvRecordValidateException(e);
-                                    }
-                                }
-                            }
-
-                            public void jsonColumn(Column column)
-                            {
-                                String v = nextColumn();
-                                if (v == null) {
-                                    pageBuilder.setNull(column);
-                                } else {
-                                    try {
-                                        pageBuilder.setJson(column, jsonParser.parse(v));
-                                    } catch (JsonParseException e) {
-                                        // TODO support default value
-                                        throw new CsvRecordValidateException(e);
-                                    }
-                                }
-                            }
-
-                            private String nextColumn()
-                            {
-                                if (allowOptionalColumns && !tokenizer.hasNextColumn()) {
-                                    //TODO warning
-                                    return null;
-                                }
-                                return tokenizer.nextColumnOrNull();
-                            }
-                        });
-
-                        try {
-                            hasNextRecord = tokenizer.nextRecord();
-                        } catch (CsvTokenizer.TooManyColumnsException ex) {
-                            if (allowExtraColumns) {
-                                String tooManyColumnsLine = tokenizer.skipCurrentLine();
-                                // TODO warning
-                                hasNextRecord = tokenizer.nextRecord();
-                            } else {
-                                // this line will be skipped at the following catch section
-                                throw ex;
-                            }
-                        }
-                        pageBuilder.addRecord();
-
-                    } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
-                        String skippedLine = tokenizer.skipCurrentLine();
-                        long lineNumber = tokenizer.getCurrentLineNumber();
-                        if (stopOnInvalidRecord) {
-                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
-                        }
-                        log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
-                        //exec.notice().skippedLine(skippedLine);
-
-                        hasNextRecord = tokenizer.nextRecord();
-                    }
-
-                    if (!hasNextRecord) {
-                        break;
-                    }
-                }
-            }
-
-            pageBuilder.finish();
-        }
-    }
-
-    static class CsvRecordValidateException
-            extends DataException
-    {
-        CsvRecordValidateException(Throwable cause)
-        {
-            super(cause);
-        }
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/CsvTokenizer.java

@@ -1,545 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.base.Preconditions;
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Deque;
-import java.util.ArrayDeque;
-import org.embulk.spi.DataException;
-import org.embulk.spi.util.LineDecoder;
-import org.embulk.config.ConfigException;
-import org.embulk.standards.CsvParserPlugin.QuotesInQuotedFields;
-
-public class CsvTokenizer
-{
-    static enum RecordState
-    {
-        NOT_END, END,
-    }
-
-    static enum ColumnState
-    {
-        BEGIN, VALUE, QUOTED_VALUE, AFTER_QUOTED_VALUE, FIRST_TRIM, LAST_TRIM_OR_VALUE,
-    }
-
-    private static final char END_OF_LINE = '\0';
-    static final char NO_QUOTE = '\0';
-    static final char NO_ESCAPE = '\0';
-
-    private final char delimiterChar;
-    private final String delimiterFollowingString;
-    private final char quote;
-    private final char escape;
-    private final String newline;
-    private final boolean trimIfNotQuoted;
-    private final QuotesInQuotedFields quotesInQuotedFields;
-    private final long maxQuotedSizeLimit;
-    private final String commentLineMarker;
-    private final LineDecoder input;
-    private final String nullStringOrNull;
-
-    private RecordState recordState = RecordState.END;  // initial state is end of a record. nextRecord() must be called first
-    private long lineNumber = 0;
-
-    private String line = null;
-    private int linePos = 0;
-    private boolean wasQuotedColumn = false;
-    private List<String> quotedValueLines = new ArrayList<>();
-    private Deque<String> unreadLines = new ArrayDeque<>();
-
-    public CsvTokenizer(LineDecoder input, CsvParserPlugin.PluginTask task)
-    {
-        String delimiter = task.getDelimiter();
-        if (delimiter.length() == 0) {
-            throw new ConfigException("Empty delimiter is not allowed");
-        } else {
-            this.delimiterChar = delimiter.charAt(0);
-            if (delimiter.length() > 1) {
-                delimiterFollowingString = delimiter.substring(1);
-            } else {
-                delimiterFollowingString = null;
-            }
-        }
-        quote = task.getQuoteChar().or(CsvParserPlugin.QuoteCharacter.noQuote()).getCharacter();
-        escape = task.getEscapeChar().or(CsvParserPlugin.EscapeCharacter.noEscape()).getCharacter();
-        newline = task.getNewline().getString();
-        trimIfNotQuoted = task.getTrimIfNotQuoted();
-        quotesInQuotedFields = task.getQuotesInQuotedFields();
-        if (trimIfNotQuoted && quotesInQuotedFields != QuotesInQuotedFields.ACCEPT_ONLY_RFC4180_ESCAPED) {
-            // The combination makes some syntax very ambiguous such as:
-            //     val1,  \"\"val2\"\"  ,val3
-            throw new ConfigException("[quotes_in_quoted_fields != ACCEPT_ONLY_RFC4180_ESCAPED] is not allowed to specify with [trim_if_not_quoted = true]");
-        }
-        maxQuotedSizeLimit = task.getMaxQuotedSizeLimit();
-        commentLineMarker = task.getCommentLineMarker().orNull();
-        nullStringOrNull = task.getNullString().orNull();
-        this.input = input;
-    }
-
-    public long getCurrentLineNumber()
-    {
-        return lineNumber;
-    }
-
-    public boolean skipHeaderLine()
-    {
-        boolean skipped = input.poll() != null;
-        if (skipped) {
-            lineNumber++;
-        }
-        return skipped;
-    }
-
-    // returns skipped line
-    public String skipCurrentLine()
-    {
-        String skippedLine;
-        if (quotedValueLines.isEmpty()) {
-            skippedLine = line;
-        } else {
-            // recover lines of quoted value
-            skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
-            unreadLines.addAll(quotedValueLines);
-            lineNumber -= quotedValueLines.size();
-            if (line != null) {
-                unreadLines.add(line);
-                lineNumber -= 1;
-            }
-            quotedValueLines.clear();
-        }
-        recordState = RecordState.END;
-        return skippedLine;
-    }
-
-    public boolean nextFile()
-    {
-        boolean next = input.nextFile();
-        if (next) {
-            lineNumber = 0;
-        }
-        return next;
-    }
-
-    // used by guess-csv
-    public boolean nextRecord()
-    {
-        return nextRecord(true);
-    }
-
-    public boolean nextRecord(boolean skipEmptyLine)
-    {
-        // If at the end of record, read the next line and initialize the state
-        if (recordState != RecordState.END) {
-            throw new TooManyColumnsException("Too many columns");
-        }
-
-        boolean hasNext = nextLine(skipEmptyLine);
-        if (hasNext) {
-            recordState = RecordState.NOT_END;
-            return true;
-        } else {
-            return false;
-        }
-    }
-
-    private boolean nextLine(boolean skipEmptyLine)
-    {
-        while (true) {
-            if (!unreadLines.isEmpty()) {
-                line = unreadLines.removeFirst();
-            } else {
-                line = input.poll();
-                if (line == null) {
-                    return false;
-                }
-            }
-            linePos = 0;
-            lineNumber++;
-
-            boolean skip = skipEmptyLine && (
-                        line.isEmpty() ||
-                        (commentLineMarker != null && line.startsWith(commentLineMarker)));
-            if (!skip) {
-                return true;
-            }
-        }
-    }
-
-    public boolean hasNextColumn()
-    {
-        return recordState == RecordState.NOT_END;
-    }
-
-    public String nextColumn()
-    {
-        if (!hasNextColumn()) {
-            throw new TooFewColumnsException("Too few columns");
-        }
-
-        // reset last state
-        wasQuotedColumn = false;
-        quotedValueLines.clear();
-
-        // local state
-        int valueStartPos = linePos;
-        int valueEndPos = 0;  // initialized by VALUE state and used by LAST_TRIM_OR_VALUE and
-        StringBuilder quotedValue = null;  // initial by VALUE or FIRST_TRIM state and used by QUOTED_VALUE state
-        ColumnState columnState = ColumnState.BEGIN;
-
-        while (true) {
-            final char c = nextChar();
-
-            switch (columnState) {
-                case BEGIN:
-                    // TODO optimization: state is BEGIN only at the first character of a column.
-                    //      this block can be out of the looop.
-                    if (isDelimiter(c)) {
-                        // empty value
-                        if (delimiterFollowingString == null) {
-                            return "";
-                        } else if (isDelimiterFollowingFrom(linePos)) {
-                            linePos += delimiterFollowingString.length();
-                            return "";
-                        }
-                        // not a delimiter
-                    }
-                    if (isEndOfLine(c)) {
-                        // empty value
-                        recordState = RecordState.END;
-                        return "";
-
-                    } else if (isSpace(c) && trimIfNotQuoted) {
-                        columnState = ColumnState.FIRST_TRIM;
-
-                    } else if (isQuote(c)) {
-                        valueStartPos = linePos;  // == 1
-                        wasQuotedColumn = true;
-                        quotedValue = new StringBuilder();
-                        columnState = ColumnState.QUOTED_VALUE;
-
-                    } else {
-                        columnState = ColumnState.VALUE;
-                    }
-                    break;
-
-                case FIRST_TRIM:
-                    if (isDelimiter(c)) {
-                        // empty value
-                        if (delimiterFollowingString == null) {
-                            return "";
-                        } else if (isDelimiterFollowingFrom(linePos)) {
-                            linePos += delimiterFollowingString.length();
-                            return "";
-                        }
-                        // not a delimiter
-                    }
-                    if (isEndOfLine(c)) {
-                        // empty value
-                        recordState = RecordState.END;
-                        return "";
-
-                    } else if (isQuote(c)) {
-                        // column has heading spaces and quoted. TODO should this be rejected?
-                        valueStartPos = linePos;
-                        wasQuotedColumn = true;
-                        quotedValue = new StringBuilder();
-                        columnState = ColumnState.QUOTED_VALUE;
-
-                    } else if (isSpace(c)) {
-                        // skip this character
-
-                    } else {
-                        valueStartPos = linePos - 1;
-                        columnState = ColumnState.VALUE;
-                    }
-                    break;
-
-                case VALUE:
-                    if (isDelimiter(c)) {
-                        if (delimiterFollowingString == null) {
-                            return line.substring(valueStartPos, linePos - 1);
-                        } else if (isDelimiterFollowingFrom(linePos)) {
-                            String value = line.substring(valueStartPos, linePos - 1);
-                            linePos += delimiterFollowingString.length();
-                            return value;
-                        }
-                        // not a delimiter
-                    }
-                    if (isEndOfLine(c)) {
-                        recordState = RecordState.END;
-                        return line.substring(valueStartPos, linePos);
-
-                    } else if (isSpace(c) && trimIfNotQuoted) {
-                        valueEndPos = linePos - 1;  // this is possibly end of value
-                        columnState = ColumnState.LAST_TRIM_OR_VALUE;
-
-                    // TODO not implemented yet foo""bar""baz -> [foo, bar, baz].append
-                    //} else if (isQuote(c)) {
-                    //    // In RFC4180, If fields are not enclosed with double quotes, then
-                    //    // double quotes may not appear inside the fields. But they are often
-                    //    // included in the fields. We should care about them later.
-
-                    } else {
-                        // keep VALUE state
-                    }
-                    break;
-
-                case LAST_TRIM_OR_VALUE:
-                    if (isDelimiter(c)) {
-                        if (delimiterFollowingString == null) {
-                            return line.substring(valueStartPos, valueEndPos);
-                        } else if (isDelimiterFollowingFrom(linePos)) {
-                            linePos += delimiterFollowingString.length();
-                            return line.substring(valueStartPos, valueEndPos);
-                        } else {
-                            // not a delimiter
-                        }
-                    }
-                    if (isEndOfLine(c)) {
-                        recordState = RecordState.END;
-                        return line.substring(valueStartPos, valueEndPos);
-
-                    } else if (isSpace(c)) {
-                        // keep LAST_TRIM_OR_VALUE state
-
-                    } else {
-                        // this spaces are not trailing spaces. go back to VALUE state
-                        columnState = ColumnState.VALUE;
-                    }
-                    break;
-
-                case QUOTED_VALUE:
-                    if (isEndOfLine(c)) {
-                        // multi-line quoted value
-                        quotedValue.append(line.substring(valueStartPos, linePos));
-                        quotedValue.append(newline);
-                        quotedValueLines.add(line);
-                        if (!nextLine(false)) {
-                            throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
-                        }
-                        valueStartPos = 0;
-
-                    } else if (isQuote(c)) {
-                        char next = peekNextChar();
-                        final char nextNext = peekNextNextChar();
-                        if (isQuote(next) &&
-                            (quotesInQuotedFields != QuotesInQuotedFields.ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS ||
-                             (!isDelimiter(nextNext) && !isEndOfLine(nextNext)))) {
-                            // Escaped by preceding it with another quote.
-                            // A quote just before a delimiter or an end of line is recognized as a functional quote,
-                            // not just as a non-escaped stray "quote character" included the field, even if
-                            // ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS is specified.
-                            quotedValue.append(line.substring(valueStartPos, linePos));
-                            valueStartPos = ++linePos;
-                        } else if (quotesInQuotedFields == QuotesInQuotedFields.ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS &&
-                            !(isDelimiter(next) || isEndOfLine(next))) {
-                            // A non-escaped stray "quote character" in the field is processed as a regular character
-                            // if ACCEPT_STRAY_QUOTES_ASSUMING_NO_DELIMITERS_IN_FIELDS is specified,
-                            if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
-                                throw new QuotedSizeLimitExceededException("The size of the quoted value exceeds the limit size (" + maxQuotedSizeLimit + ")");
-                            }
-                        } else {
-                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
-                            columnState = ColumnState.AFTER_QUOTED_VALUE;
-                        }
-
-                    } else if (isEscape(c)) {  // isQuote must be checked first in case of quote == escape
-                        // In RFC 4180, CSV's escape char is '\"'. But '\\' is often used.
-                        char next = peekNextChar();
-                        if (isEndOfLine(c)) {
-                            // escape end of line. TODO assuming multi-line quoted value without newline?
-                            quotedValue.append(line.substring(valueStartPos, linePos));
-                            quotedValueLines.add(line);
-                            if (!nextLine(false)) {
-                                throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
-                            }
-                            valueStartPos = 0;
-                        } else if (isQuote(next) || isEscape(next)) { // escaped quote
-                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
-                            quotedValue.append(next);
-                            valueStartPos = ++linePos;
-                        }
-
-                    } else {
-                        if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
-                            throw new QuotedSizeLimitExceededException("The size of the quoted value exceeds the limit size ("+maxQuotedSizeLimit+")");
-                        }
-                        // keep QUOTED_VALUE state
-                    }
-                    break;
-
-                case AFTER_QUOTED_VALUE:
-                    if (isDelimiter(c)) {
-                        if (delimiterFollowingString == null) {
-                            return quotedValue.toString();
-                        } else if (isDelimiterFollowingFrom(linePos)) {
-                            linePos += delimiterFollowingString.length();
-                            return quotedValue.toString();
-                        }
-                        // not a delimiter
-                    }
-                    if (isEndOfLine(c)) {
-                        recordState = RecordState.END;
-                        return quotedValue.toString();
-
-                    } else if (isSpace(c)) {
-                        // column has trailing spaces and quoted. TODO should this be rejected?
-
-                    } else {
-                        throw new InvalidValueException(String.format("Unexpected extra character '%c' after a value quoted by '%c'", c, quote));
-                    }
-                    break;
-
-                default:
-                    assert false;
-            }
-        }
-    }
-
-    public String nextColumnOrNull()
-    {
-        String v = nextColumn();
-        if (nullStringOrNull == null) {
-            if (v.isEmpty()) {
-                if (wasQuotedColumn) {
-                    return "";
-                }
-                else {
-                    return null;
-                }
-            }
-            else {
-                return v;
-            }
-        }
-        else {
-            if (v.equals(nullStringOrNull)) {
-                return null;
-            }
-            else {
-                return v;
-            }
-        }
-    }
-
-    public boolean wasQuotedColumn()
-    {
-        return wasQuotedColumn;
-    }
-
-    private char nextChar()
-    {
-        Preconditions.checkState(line != null, "nextColumn is called after end of file");
-
-        if (linePos >= line.length()) {
-            return END_OF_LINE;
-        } else {
-            return line.charAt(linePos++);
-        }
-    }
-
-    private char peekNextChar()
-    {
-        Preconditions.checkState(line != null, "peekNextChar is called after end of file");
-
-        if (linePos >= line.length()) {
-            return END_OF_LINE;
-        } else {
-            return line.charAt(linePos);
-        }
-    }
-
-    private char peekNextNextChar()
-    {
-        Preconditions.checkState(line != null, "peekNextNextChar is called after end of file");
-
-        if (linePos + 1 >= line.length()) {
-            return END_OF_LINE;
-        } else {
-            return line.charAt(linePos + 1);
-        }
-    }
-
-    private boolean isSpace(char c)
-    {
-        return c == ' ';
-    }
-
-    private boolean isDelimiterFollowingFrom(int pos)
-    {
-        if (line.length() < pos + delimiterFollowingString.length()) {
-            return false;
-        }
-        for (int i = 0; i < delimiterFollowingString.length(); i++) {
-            if (delimiterFollowingString.charAt(i) != line.charAt(pos + i)) {
-                return false;
-            }
-        }
-        return true;
-    }
-
-    private boolean isDelimiter(char c)
-    {
-        return c == delimiterChar;
-    }
-
-    private boolean isEndOfLine(char c)
-    {
-        return c == END_OF_LINE;
-    }
-
-    private boolean isQuote(char c)
-    {
-        return quote != NO_QUOTE && c == quote;
-    }
-
-    private boolean isEscape(char c)
-    {
-        return escape != NO_ESCAPE && c == escape;
-    }
-
-    public static class InvalidFormatException
-            extends DataException
-    {
-        public InvalidFormatException(String message)
-        {
-            super(message);
-        }
-    }
-
-    public static class InvalidValueException
-            extends DataException
-    {
-        public InvalidValueException(String message)
-        {
-            super(message);
-        }
-    }
-
-    public static class QuotedSizeLimitExceededException
-            extends InvalidValueException
-    {
-        public QuotedSizeLimitExceededException(String message)
-        {
-            super(message);
-        }
-    }
-
-    public class TooManyColumnsException
-            extends InvalidFormatException
-    {
-        public TooManyColumnsException(String message)
-        {
-            super(message);
-        }
-    }
-
-    public class TooFewColumnsException
-            extends InvalidFormatException
-    {
-        public TooFewColumnsException(String message)
-        {
-            super(message);
-        }
-    }
-}