embulk-parser-csv_with_default_value 0.1.0

data/gradlew.bat

@@ -0,0 +1,90 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/guess/csv_with_default_value.rb

@@ -0,0 +1,61 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g "csv_with_default_value" partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+
+    #class CsvWithDefaultValue < GuessPlugin
+    #  Plugin.register_guess("csv_with_default_value", self)
+    #
+    #  def guess(config, sample_buffer)
+    #    if sample_buffer[0,2] == GZIP_HEADER
+    #      guessed = {}
+    #      guessed["getType"] = "csv_with_default_value"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class CsvWithDefaultValue < TextGuessPlugin
+    #  Plugin.register_guess("csv_with_default_value", self)
+    #
+    #  def guess_text(config, sample_text)
+    #    js = JSON.parse(sample_text) rescue nil
+    #    if js && js["mykeyword"] == "keyword"
+    #      guessed = {}
+    #      guessed["getType"] = "csv_with_default_value"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+    #class CsvWithDefaultValue < LineGuessPlugin
+    #  Plugin.register_guess("csv_with_default_value", self)
+    #
+    #  def guess_lines(config, sample_lines)
+    #    all_line_matched = sample_lines.all? do |line|
+    #      line =~ /mypattern/
+    #    end
+    #    if all_line_matched
+    #      guessed = {}
+    #      guessed["getType"] = "csv_with_default_value"
+    #      guessed["property1"] = "guessed-value"
+    #      return {"parser" => guessed}
+    #    else
+    #      return {}
+    #    end
+    #  end
+    #end
+
+  end
+end

data/lib/embulk/parser/csv_with_default_value.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "csv_with_default_value", "org.embulk.parser.csv_with_default_value.CsvWithDefaultValueParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/parser/csv_with_default_value/ColumnDefaultValue.java

@@ -0,0 +1,123 @@
+package org.embulk.parser.csv_with_default_value;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.google.common.base.Joiner;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.type.Types;
+
+import java.util.Locale;
+import java.util.Map;
+import java.util.Set;
+
+@JsonDeserialize(as = ColumnDefaultValueImpl.class)
+public interface ColumnDefaultValue {
+
+    Set<Type> ALLOWED_TYPES = ImmutableSet.<Type>of(Types.LONG, Types.DOUBLE, Types.DOUBLE, Types.TIMESTAMP);
+    String ALLOWED_TYPES_NAME = Joiner.on(",").join(ALLOWED_TYPES);
+
+    @Config("default_value")
+    Optional<String> getDefaultValue();
+
+    @Config("type")
+    @ConfigDefault("immediate")
+    ColumnDefaultValue.ValueType getType();
+
+    interface DefaultValueSetter {
+
+        /**
+         * @throws CsvRecordValidateException
+         */
+        void longValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column);
+
+        /**
+         * @throws CsvRecordValidateException
+         */
+        void doubleValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column);
+
+        /**
+         * @throws CsvRecordValidateException
+         */
+        void timestampValue(ColumnDefaultValue value, TimestampParser parser, PageBuilder pageBuilder, Column column);
+
+    }
+
+    enum ValueType implements DefaultValueSetter {
+        IMMEDIATE {
+            @Override
+            public void longValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column) {
+                try {
+                    pageBuilder.setLong(column, Long.parseLong(value.getDefaultValue().get()));
+                } catch (NumberFormatException e) {
+                    throw new CsvRecordValidateException(e);
+                }
+            }
+
+            @Override
+            public void doubleValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column) {
+                try {
+                    pageBuilder.setDouble(column, Double.parseDouble(value.getDefaultValue().get()));
+                } catch (NumberFormatException e) {
+                    throw new CsvRecordValidateException(e);
+                }
+
+            }
+
+            @Override
+            public void timestampValue(ColumnDefaultValue value, TimestampParser parser, PageBuilder pageBuilder, Column column) {
+                try {
+                    pageBuilder.setTimestamp(column, parser.parse(value.getDefaultValue().get()));
+                } catch (TimestampParseException e) {
+                    throw new CsvRecordValidateException(e);
+                }
+            }
+        },
+        NULL {
+            @Override
+            public void longValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column) {
+                throw new ConfigException("null value is not allowed for long");
+            }
+
+            @Override
+            public void doubleValue(ColumnDefaultValue value, PageBuilder pageBuilder, Column column) {
+                throw new ConfigException("null value is not allowed for double");
+            }
+
+            @Override
+            public void timestampValue(ColumnDefaultValue value, TimestampParser parser, PageBuilder pageBuilder, Column column) {
+                pageBuilder.setNull(column);
+            }
+        };
+
+        @JsonValue
+        @Override
+        public String toString() {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+
+        @JsonCreator
+        public static ValueType fromString(String value) {
+            Map<String, ValueType> types = ImmutableMap.of(IMMEDIATE.toString().toLowerCase(), IMMEDIATE, NULL.toString().toLowerCase(), NULL);
+            ValueType type = types.get(value);
+            if (type != null) {
+                return type;
+            } else {
+                throw new ConfigException(String.format("Unknown value_type '%s', Supported getType are immediate, null.", value));
+            }
+        }
+    }
+}
+
+

data/src/main/java/org/embulk/parser/csv_with_default_value/ColumnDefaultValueImpl.java

@@ -0,0 +1,68 @@
+package org.embulk.parser.csv_with_default_value;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+
+public class ColumnDefaultValueImpl implements ColumnDefaultValue {
+
+    @JsonProperty("default_value")
+    private Optional<String> defaultValue;
+
+    @JsonProperty("type")
+    private ValueType type;
+
+    public ColumnDefaultValueImpl(){
+        this(Optional.<String>absent(), ValueType.IMMEDIATE);
+    }
+
+    public ColumnDefaultValueImpl(String defaultValue, ValueType type) {
+        this(Optional.of(defaultValue), type);
+    }
+
+    public ColumnDefaultValueImpl(Optional<String> defaultValue, ValueType type){
+        this.defaultValue = defaultValue;
+        this.type = type;
+    }
+
+    @Override
+    public Optional<String> getDefaultValue() {
+        return defaultValue;
+    }
+
+    @Override
+    public ValueType getType() {
+        return type;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || ! (o instanceof ColumnDefaultValue)) return false;
+
+        ColumnDefaultValue that = (ColumnDefaultValue) o;
+
+        if(getDefaultValue().isPresent() != that.getDefaultValue().isPresent()){
+            return false;
+        }else if(getDefaultValue().isPresent() && that.getDefaultValue().isPresent()){
+            if(!getDefaultValue().get().equals(that.getDefaultValue().get())){
+                return false;
+            }
+            return getType() == that.getType();
+        }else{
+            return getType() == that.getType();
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        int result = getDefaultValue() != null ? getDefaultValue().hashCode() : 0;
+        result = 31 * result + (getType() != null ? getType().hashCode() : 0);
+        return result;
+    }
+
+    @Override
+    public String toString(){
+        return String.format("ColumnDefaultValueImpl(type=%s,value=%s)", getType(), getDefaultValue().or("null"));
+    }
+
+}

data/src/main/java/org/embulk/parser/csv_with_default_value/CsvRecordValidateException.java

@@ -0,0 +1,13 @@
+package org.embulk.parser.csv_with_default_value;
+
+import org.embulk.spi.DataException;
+
+/**
+ * Created by k.sasaki on 2016/10/25.
+ */
+class CsvRecordValidateException
+        extends DataException {
+    CsvRecordValidateException(Throwable cause) {
+        super(cause);
+    }
+}

data/src/main/java/org/embulk/parser/csv_with_default_value/CsvTokenizer.java

@@ -0,0 +1,512 @@
+package org.embulk.parser.csv_with_default_value;
+
+import com.google.common.base.Preconditions;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Deque;
+import java.util.ArrayDeque;
+import org.embulk.spi.DataException;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.config.ConfigException;
+
+public class CsvTokenizer
+{
+    enum RecordState
+    {
+        NOT_END, END,
+    }
+
+    enum ColumnState
+    {
+        BEGIN, VALUE, QUOTED_VALUE, AFTER_QUOTED_VALUE, FIRST_TRIM, LAST_TRIM_OR_VALUE,
+    }
+
+    private static final char END_OF_LINE = '\0';
+    static final char NO_QUOTE = '\0';
+    static final char NO_ESCAPE = '\0';
+
+    private final char delimiterChar;
+    private final String delimiterFollowingString;
+    private final char quote;
+    private final char escape;
+    private final String newline;
+    private final boolean trimIfNotQuoted;
+    private final long maxQuotedSizeLimit;
+    private final String commentLineMarker;
+    private final LineDecoder input;
+    private final String nullStringOrNull;
+
+    private RecordState recordState = RecordState.END;  // initial state is end of a record. nextRecord() must be called first
+    private long lineNumber = 0;
+
+    private String line = null;
+    private int linePos = 0;
+    private boolean wasQuotedColumn = false;
+    private List<String> quotedValueLines = new ArrayList<>();
+    private Deque<String> unreadLines = new ArrayDeque<>();
+
+    public CsvTokenizer(LineDecoder input, CsvWithDefaultValueParserPlugin.PluginTask task)
+    {
+        String delimiter = task.getDelimiter();
+        if (delimiter.length() == 0) {
+            throw new ConfigException("Empty delimiter is not allowed");
+        } else {
+            this.delimiterChar = delimiter.charAt(0);
+            if (delimiter.length() > 1) {
+                delimiterFollowingString = delimiter.substring(1);
+            } else {
+                delimiterFollowingString = null;
+            }
+        }
+        quote = task.getQuoteChar().or(CsvWithDefaultValueParserPlugin.QuoteCharacter.noQuote()).getCharacter();
+        escape = task.getEscapeChar().or(CsvWithDefaultValueParserPlugin.EscapeCharacter.noEscape()).getCharacter();
+        newline = task.getNewline().getString();
+        trimIfNotQuoted = task.getTrimIfNotQuoted();
+        maxQuotedSizeLimit = task.getMaxQuotedSizeLimit();
+        commentLineMarker = task.getCommentLineMarker().orNull();
+        nullStringOrNull = task.getNullString().orNull();
+        this.input = input;
+    }
+
+    public long getCurrentLineNumber()
+    {
+        return lineNumber;
+    }
+
+    public boolean skipHeaderLine()
+    {
+        boolean skipped = input.poll() != null;
+        if (skipped) {
+            lineNumber++;
+        }
+        return skipped;
+    }
+
+    // returns skipped line
+    public String skipCurrentLine()
+    {
+        String skippedLine;
+        if (quotedValueLines.isEmpty()) {
+            skippedLine = line;
+        } else {
+            // recover lines of quoted value
+            skippedLine = quotedValueLines.remove(0);  // TODO optimize performance
+            unreadLines.addAll(quotedValueLines);
+            lineNumber -= quotedValueLines.size();
+            if (line != null) {
+                unreadLines.add(line);
+                lineNumber -= 1;
+            }
+            quotedValueLines.clear();
+        }
+        recordState = RecordState.END;
+        return skippedLine;
+    }
+
+    public boolean nextFile()
+    {
+        boolean next = input.nextFile();
+        if (next) {
+            lineNumber = 0;
+        }
+        return next;
+    }
+
+    // used by guess-csv
+    public boolean nextRecord()
+    {
+        return nextRecord(true);
+    }
+
+    public boolean nextRecord(boolean skipEmptyLine)
+    {
+        // If at the end of record, read the next line and initialize the state
+        if (recordState != RecordState.END) {
+            throw new TooManyColumnsException("Too many columns");
+        }
+
+        boolean hasNext = nextLine(skipEmptyLine);
+        if (hasNext) {
+            recordState = RecordState.NOT_END;
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    private boolean nextLine(boolean skipEmptyLine)
+    {
+        while (true) {
+            if (!unreadLines.isEmpty()) {
+                line = unreadLines.removeFirst();
+            } else {
+                line = input.poll();
+                if (line == null) {
+                    return false;
+                }
+            }
+            linePos = 0;
+            lineNumber++;
+
+            boolean skip = skipEmptyLine && (
+                    line.isEmpty() ||
+                            (commentLineMarker != null && line.startsWith(commentLineMarker)));
+            if (!skip) {
+                return true;
+            }
+        }
+    }
+
+    public boolean hasNextColumn()
+    {
+        return recordState == RecordState.NOT_END;
+    }
+
+    public String nextColumn()
+    {
+        if (!hasNextColumn()) {
+            throw new TooFewColumnsException("Too few columns");
+        }
+
+        // reset last state
+        wasQuotedColumn = false;
+        quotedValueLines.clear();
+
+        // local state
+        int valueStartPos = linePos;
+        int valueEndPos = 0;  // initialized by VALUE state and used by LAST_TRIM_OR_VALUE and
+        StringBuilder quotedValue = null;  // initial by VALUE or FIRST_TRIM state and used by QUOTED_VALUE state
+        ColumnState columnState = ColumnState.BEGIN;
+
+        while (true) {
+            final char c = nextChar();
+
+            switch (columnState) {
+                case BEGIN:
+                    // TODO optimization: state is BEGIN only at the first character of a column.
+                    //      this block can be out of the looop.
+                    if (isDelimiter(c)) {
+                        // empty value
+                        if (delimiterFollowingString == null) {
+                            return "";
+                        } else if (isDelimiterFollowingFrom(linePos)) {
+                            linePos += delimiterFollowingString.length();
+                            return "";
+                        }
+                        // not a delimiter
+                    }
+                    if (isEndOfLine(c)) {
+                        // empty value
+                        recordState = RecordState.END;
+                        return "";
+
+                    } else if (isSpace(c) && trimIfNotQuoted) {
+                        columnState = ColumnState.FIRST_TRIM;
+
+                    } else if (isQuote(c)) {
+                        valueStartPos = linePos;  // == 1
+                        wasQuotedColumn = true;
+                        quotedValue = new StringBuilder();
+                        columnState = ColumnState.QUOTED_VALUE;
+
+                    } else {
+                        columnState = ColumnState.VALUE;
+                    }
+                    break;
+
+                case FIRST_TRIM:
+                    if (isDelimiter(c)) {
+                        // empty value
+                        if (delimiterFollowingString == null) {
+                            return "";
+                        } else if (isDelimiterFollowingFrom(linePos)) {
+                            linePos += delimiterFollowingString.length();
+                            return "";
+                        }
+                        // not a delimiter
+                    }
+                    if (isEndOfLine(c)) {
+                        // empty value
+                        recordState = RecordState.END;
+                        return "";
+
+                    } else if (isQuote(c)) {
+                        // column has heading spaces and quoted. TODO should this be rejected?
+                        valueStartPos = linePos;
+                        wasQuotedColumn = true;
+                        quotedValue = new StringBuilder();
+                        columnState = ColumnState.QUOTED_VALUE;
+
+                    } else if (isSpace(c)) {
+                        // skip this character
+
+                    } else {
+                        valueStartPos = linePos - 1;
+                        columnState = ColumnState.VALUE;
+                    }
+                    break;
+
+                case VALUE:
+                    if (isDelimiter(c)) {
+                        if (delimiterFollowingString == null) {
+                            return line.substring(valueStartPos, linePos - 1);
+                        } else if (isDelimiterFollowingFrom(linePos)) {
+                            String value = line.substring(valueStartPos, linePos - 1);
+                            linePos += delimiterFollowingString.length();
+                            return value;
+                        }
+                        // not a delimiter
+                    }
+                    if (isEndOfLine(c)) {
+                        recordState = RecordState.END;
+                        return line.substring(valueStartPos, linePos);
+
+                    } else if (isSpace(c) && trimIfNotQuoted) {
+                        valueEndPos = linePos - 1;  // this is possibly end of value
+                        columnState = ColumnState.LAST_TRIM_OR_VALUE;
+
+                        // TODO not implemented yet foo""bar""baz -> [foo, bar, baz].append
+                        //} else if (isQuote(c)) {
+                        //    // In RFC4180, If fields are not enclosed with double quotes, then
+                        //    // double quotes may not appear inside the fields. But they are often
+                        //    // included in the fields. We should care about them later.
+
+                    } else {
+                        // keep VALUE state
+                    }
+                    break;
+
+                case LAST_TRIM_OR_VALUE:
+                    if (isDelimiter(c)) {
+                        if (delimiterFollowingString == null) {
+                            return line.substring(valueStartPos, valueEndPos);
+                        } else if (isDelimiterFollowingFrom(linePos)) {
+                            linePos += delimiterFollowingString.length();
+                            return line.substring(valueStartPos, valueEndPos);
+                        } else {
+                            // not a delimiter
+                        }
+                    }
+                    if (isEndOfLine(c)) {
+                        recordState = RecordState.END;
+                        return line.substring(valueStartPos, valueEndPos);
+
+                    } else if (isSpace(c)) {
+                        // keep LAST_TRIM_OR_VALUE state
+
+                    } else {
+                        // this spaces are not trailing spaces. go back to VALUE state
+                        columnState = ColumnState.VALUE;
+                    }
+                    break;
+
+                case QUOTED_VALUE:
+                    if (isEndOfLine(c)) {
+                        // multi-line quoted value
+                        quotedValue.append(line.substring(valueStartPos, linePos));
+                        quotedValue.append(newline);
+                        quotedValueLines.add(line);
+                        if (!nextLine(false)) {
+                            throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
+                        }
+                        valueStartPos = 0;
+
+                    } else if (isQuote(c)) {
+                        char next = peekNextChar();
+                        if (isQuote(next)) { // escaped quote
+                            quotedValue.append(line.substring(valueStartPos, linePos));
+                            valueStartPos = ++linePos;
+                        } else {
+                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
+                            columnState = ColumnState.AFTER_QUOTED_VALUE;
+                        }
+
+                    } else if (isEscape(c)) {  // isQuote must be checked first in case of quote == escape
+                        // In RFC 4180, CSV's escape char is '\"'. But '\\' is often used.
+                        char next = peekNextChar();
+                        if (isEndOfLine(c)) {
+                            // escape end of line. TODO assuming multi-line quoted value without newline?
+                            quotedValue.append(line.substring(valueStartPos, linePos));
+                            quotedValueLines.add(line);
+                            if (!nextLine(false)) {
+                                throw new InvalidValueException("Unexpected end of line during parsing a quoted value");
+                            }
+                            valueStartPos = 0;
+                        } else if (isQuote(next) || isEscape(next)) { // escaped quote
+                            quotedValue.append(line.substring(valueStartPos, linePos - 1));
+                            quotedValue.append(next);
+                            valueStartPos = ++linePos;
+                        }
+
+                    } else {
+                        if ((linePos - valueStartPos) + quotedValue.length() > maxQuotedSizeLimit) {
+                            throw new QuotedSizeLimitExceededException("The size of the quoted value exceeds the limit size ("+maxQuotedSizeLimit+")");
+                        }
+                        // keep QUOTED_VALUE state
+                    }
+                    break;
+
+                case AFTER_QUOTED_VALUE:
+                    if (isDelimiter(c)) {
+                        if (delimiterFollowingString == null) {
+                            return quotedValue.toString();
+                        } else if (isDelimiterFollowingFrom(linePos)) {
+                            linePos += delimiterFollowingString.length();
+                            return quotedValue.toString();
+                        }
+                        // not a delimiter
+                    }
+                    if (isEndOfLine(c)) {
+                        recordState = RecordState.END;
+                        return quotedValue.toString();
+
+                    } else if (isSpace(c)) {
+                        // column has trailing spaces and quoted. TODO should this be rejected?
+
+                    } else {
+                        throw new InvalidValueException(String.format("Unexpected extra character '%c' after a value quoted by '%c'", c, quote));
+                    }
+                    break;
+
+                default:
+                    assert false;
+            }
+        }
+    }
+
+    public String nextColumnOrNull()
+    {
+        String v = nextColumn();
+        if (nullStringOrNull == null) {
+            if (v.isEmpty()) {
+                if (wasQuotedColumn) {
+                    return "";
+                }
+                else {
+                    return null;
+                }
+            }
+            else {
+                return v;
+            }
+        }
+        else {
+            if (v.equals(nullStringOrNull)) {
+                return null;
+            }
+            else {
+                return v;
+            }
+        }
+    }
+
+    public boolean wasQuotedColumn()
+    {
+        return wasQuotedColumn;
+    }
+
+    private char nextChar()
+    {
+        Preconditions.checkState(line != null, "nextColumn is called after end of file");
+
+        if (linePos >= line.length()) {
+            return END_OF_LINE;
+        } else {
+            return line.charAt(linePos++);
+        }
+    }
+
+    private char peekNextChar()
+    {
+        Preconditions.checkState(line != null, "peekNextChar is called after end of file");
+
+        if (linePos >= line.length()) {
+            return END_OF_LINE;
+        } else {
+            return line.charAt(linePos);
+        }
+    }
+
+    private boolean isSpace(char c)
+    {
+        return c == ' ';
+    }
+
+    private boolean isDelimiterFollowingFrom(int pos)
+    {
+        if (line.length() < pos + delimiterFollowingString.length()) {
+            return false;
+        }
+        for (int i = 0; i < delimiterFollowingString.length(); i++) {
+            if (delimiterFollowingString.charAt(i) != line.charAt(pos + i)) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private boolean isDelimiter(char c)
+    {
+        return c == delimiterChar;
+    }
+
+    private boolean isEndOfLine(char c)
+    {
+        return c == END_OF_LINE;
+    }
+
+    private boolean isQuote(char c)
+    {
+        return quote != NO_QUOTE && c == quote;
+    }
+
+    private boolean isEscape(char c)
+    {
+        return escape != NO_ESCAPE && c == escape;
+    }
+
+    public static class InvalidFormatException
+            extends DataException
+    {
+        public InvalidFormatException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public static class InvalidValueException
+            extends DataException
+    {
+        public InvalidValueException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public static class QuotedSizeLimitExceededException
+            extends InvalidValueException
+    {
+        public QuotedSizeLimitExceededException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public class TooManyColumnsException
+            extends InvalidFormatException
+    {
+        public TooManyColumnsException(String message)
+        {
+            super(message);
+        }
+    }
+
+    public class TooFewColumnsException
+            extends InvalidFormatException
+    {
+        public TooFewColumnsException(String message)
+        {
+            super(message);
+        }
+    }
+}