embulk-parser-csv_guessable 0.1.1

data/gradlew.bat

@@ -0,0 +1,84 @@
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/lib/embulk/guess/csv_guessable.rb

@@ -0,0 +1,61 @@
+module Embulk
+  module Guess
+
+    # TODO implement guess plugin to make this command work:
+    #      $ embulk guess -g "csv_guessable" partial-config.yml
+    #
+    #      Depending on the file format the plugin uses, you can use choose
+    #      one of binary guess (GuessPlugin), text guess (TextGuessPlugin),
+    #      or line guess (LineGuessPlugin).
+
+    # class CsvGuessable < GuessPlugin
+    #   Plugin.register_guess("csv_guessable", self)
+    #
+    #   def guess(config, sample_buffer)
+    #     if sample_buffer[0,2] == GZIP_HEADER
+    #       guessed = {}
+    #       guessed["type"] = "csv_guessable"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class CsvGuessable < TextGuessPlugin
+    #   Plugin.register_guess("csv_guessable", self)
+    #
+    #   def guess_text(config, sample_text)
+    #     js = JSON.parse(sample_text) rescue nil
+    #     if js && js["mykeyword"] == "keyword"
+    #       guessed = {}
+    #       guessed["type"] = "csv_guessable"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+    # class CsvGuessable < LineGuessPlugin
+    #   Plugin.register_guess("csv_guessable", self)
+    #
+    #   def guess_lines(config, sample_lines)
+    #     all_line_matched = sample_lines.all? do |line|
+    #       line =~ /mypattern/
+    #     end
+    #     if all_line_matched
+    #       guessed = {}
+    #       guessed["type"] = "csv_guessable"
+    #       guessed["property1"] = "guessed-value"
+    #       return {"parser" => guessed}
+    #     else
+    #       return {}
+    #     end
+    #   end
+    # end
+
+  end
+end

data/lib/embulk/parser/csv_guessable.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_parser(
+  "csv_guessable", "org.embulk.parser.csv_guessable.CsvGuessableParserPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/parser/csv_guessable/CsvGuessableParserPlugin.java

@@ -0,0 +1,371 @@
+package org.embulk.parser.csv_guessable;
+
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableSet;
+import java.io.BufferedReader;
+import com.opencsv.CSVReader; // TODO: use embulk's parser
+import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import org.slf4j.Logger;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.DataException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.json.JsonParser;
+import org.embulk.spi.json.JsonParseException;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.type.Types;
+import org.embulk.spi.unit.LocalFile;
+import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.util.Timestamps;
+
+import org.embulk.standards.CsvParserPlugin;
+
+public class CsvGuessableParserPlugin
+        extends CsvParserPlugin
+{
+    private static final ImmutableSet<String> TRUE_STRINGS =
+        ImmutableSet.of(
+                "true", "True", "TRUE",
+                "yes", "Yes", "YES",
+                "t", "T", "y", "Y",
+                "on", "On", "ON",
+                "1");
+
+    private final Logger log;
+
+    public CsvGuessableParserPlugin()
+    {
+        log = Exec.getLogger(CsvGuessableParserPlugin.class);
+    }
+
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.Task
+    {
+        @Config("columns")
+        @ConfigDefault("null")
+        Optional<SchemaConfig> getSchemaConfig();
+
+        @Config("header_line")
+        @ConfigDefault("null")
+        Optional<Boolean> getHeaderLine();
+
+        @Config("skip_header_lines")
+        @ConfigDefault("0")
+        int getSkipHeaderLines();
+        void setSkipHeaderLines(int n);
+
+        @Config("delimiter")
+        @ConfigDefault("\",\"")
+        String getDelimiter();
+
+        @Config("quote")
+        @ConfigDefault("\"\\\"\"")
+        Optional<QuoteCharacter> getQuoteChar();
+
+        @Config("escape")
+        @ConfigDefault("\"\\\\\"")
+        Optional<EscapeCharacter> getEscapeChar();
+
+        // Null value handling: if the CsvParser found 'non-quoted empty string's,
+        // it replaces them to string that users specified like "\N", "NULL".
+        @Config("null_string")
+        @ConfigDefault("null")
+        Optional<String> getNullString();
+
+        @Config("trim_if_not_quoted")
+        @ConfigDefault("false")
+        boolean getTrimIfNotQuoted();
+
+        @Config("max_quoted_size_limit")
+        @ConfigDefault("131072") //128kB
+        long getMaxQuotedSizeLimit();
+
+        @Config("comment_line_marker")
+        @ConfigDefault("null")
+        Optional<String> getCommentLineMarker();
+
+        @Config("allow_optional_columns")
+        @ConfigDefault("false")
+        boolean getAllowOptionalColumns();
+
+        @Config("allow_extra_columns")
+        @ConfigDefault("false")
+        boolean getAllowExtraColumns();
+
+        @Config("stop_on_invalid_record")
+        @ConfigDefault("false")
+        boolean getStopOnInvalidRecord();
+
+        @Config("schema_file")
+        @ConfigDefault("null")
+        public Optional<LocalFile> getSchemaFile();
+
+        @Config("schema_line")
+        @ConfigDefault("1")
+        public int getSchemaLine();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        SchemaConfig schemaConfig = null;
+
+        if (task.getSchemaFile().isPresent()) { /* embulk-parser-csv_guessable */
+            if (task.getHeaderLine().isPresent()) {
+                // TODO: use 'columns' as hints for guess
+                throw new ConfigException("embulk-parsre-csv_gussable will use 'columnes' as hints for guess as hints for guess. Please delete 'columnes' now.");
+            } else { /* guess from header */
+                int schemaLine = task.getSchemaLine();
+                task.setSkipHeaderLines(schemaLine); // TODO: use 'skip_header_line'
+
+                String header = readHeader(task.getSchemaFile().get().getPath(), schemaLine);
+                log.debug(header);
+                ArrayList<ColumnConfig> columns = newColumns(header, config);
+                log.debug(columns.toString());
+                schemaConfig = new SchemaConfig(columns);
+            }
+        } else { /* embulk-parser-csv embulk */
+            // backward compatibility
+            if (task.getHeaderLine().isPresent()) {
+                if (task.getSkipHeaderLines() > 0) {
+                    throw new ConfigException("'header_line' option is invalid if 'skip_header_lines' is set.");
+                }
+                if (task.getHeaderLine().get()) {
+                    task.setSkipHeaderLines(1);
+                } else {
+                    task.setSkipHeaderLines(0);
+                }
+            }
+            schemaConfig = task.getSchemaConfig().get();
+        }
+
+        control.run(task.dump(), schemaConfig.toSchema());
+    }
+
+    @Override
+    public void run(TaskSource taskSource, final Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        TimestampParser[] timestampParsers = null;
+        if (task.getSchemaConfig().isPresent()) {
+          timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig().get());
+        }
+        final JsonParser jsonParser = new JsonParser();
+        final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task);
+        final boolean allowOptionalColumns = task.getAllowOptionalColumns();
+        final boolean allowExtraColumns = task.getAllowExtraColumns();
+        final boolean stopOnInvalidRecord = task.getStopOnInvalidRecord();
+        int skipHeaderLines = task.getSkipHeaderLines();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            while (tokenizer.nextFile()) {
+                // skip the header lines for each file
+                for (; skipHeaderLines > 0; skipHeaderLines--) {
+                    if (!tokenizer.skipHeaderLine()) {
+                        break;
+                    }
+                }
+
+                if (!tokenizer.nextRecord()) {
+                    // empty file
+                    continue;
+                }
+
+                while (true) {
+                    boolean hasNextRecord;
+
+                    try {
+                        schema.visitColumns(new ColumnVisitor() {
+                            public void booleanColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    pageBuilder.setBoolean(column, TRUE_STRINGS.contains(v));
+                                }
+                            }
+
+                            public void longColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setLong(column, Long.parseLong(v));
+                                    } catch (NumberFormatException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            public void doubleColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setDouble(column, Double.parseDouble(v));
+                                    } catch (NumberFormatException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            public void stringColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    pageBuilder.setString(column, v);
+                                }
+                            }
+
+                            public void timestampColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+//                                        pageBuilder.setTimestamp(column, timestampParsers[column.getIndex()].parse(v));
+                                    } catch (TimestampParseException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            public void jsonColumn(Column column)
+                            {
+                                String v = nextColumn();
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setJson(column, jsonParser.parse(v));
+                                    } catch (JsonParseException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            private String nextColumn()
+                            {
+                                if (allowOptionalColumns && !tokenizer.hasNextColumn()) {
+                                    //TODO warning
+                                    return null;
+                                }
+                                return tokenizer.nextColumnOrNull();
+                            }
+                        });
+
+                        try {
+                            hasNextRecord = tokenizer.nextRecord();
+                        } catch (CsvTokenizer.TooManyColumnsException ex) {
+                            if (allowExtraColumns) {
+                                String tooManyColumnsLine = tokenizer.skipCurrentLine();
+                                // TODO warning
+                                hasNextRecord = tokenizer.nextRecord();
+                            } else {
+                                // this line will be skipped at the following catch section
+                                throw ex;
+                            }
+                        }
+                        pageBuilder.addRecord();
+
+                    } catch (CsvTokenizer.InvalidFormatException | CsvTokenizer.InvalidValueException | CsvRecordValidateException e) {
+                        String skippedLine = tokenizer.skipCurrentLine();
+                        long lineNumber = tokenizer.getCurrentLineNumber();
+                        if (stopOnInvalidRecord) {
+                            throw new DataException(String.format("Invalid record at line %d: %s", lineNumber, skippedLine), e);
+                        }
+                        log.warn(String.format("Skipped line %d (%s): %s", lineNumber, e.getMessage(), skippedLine));
+                        //exec.notice().skippedLine(skippedLine);
+
+                        hasNextRecord = tokenizer.nextRecord();
+                    }
+
+                    if (!hasNextRecord) {
+                        break;
+                    }
+                }
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    static class CsvRecordValidateException
+            extends DataException
+    {
+        CsvRecordValidateException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+
+    private String readHeader(Path path, int schemaLine) {
+        if (schemaLine <= 0) {
+            throw new ConfigException("'schemaLine' must be set '> 0'");
+        }
+
+        String line = null;
+        try (BufferedReader br = Files.newBufferedReader(path, StandardCharsets.UTF_8)) {
+            for (int i=1; i <= schemaLine; ++i) {
+                line = br.readLine();
+                if (line == null) {
+                    throw new ConfigException("not found 'schema_line' in 'schema_file'");
+                }
+            }
+        } catch (IOException e) {
+            throw new ConfigException(e);
+        }
+        return line;
+    }
+
+    private ArrayList<ColumnConfig> newColumns(String header, ConfigSource config) {
+        ArrayList columns = new ArrayList<ArrayList>(); 
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        try (CSVReader reader = new CSVReader(new StringReader(header))) {
+            String[] csv = reader.readNext();
+            for (String column: csv) {
+                columns.add(new ColumnConfig(column, Types.STRING, config));
+            }
+        } catch (IOException e) {
+            throw new ConfigException(e);
+        }
+
+        return columns;
+    }
+}