embulk 0.1.0

data/embulk-standards/build.gradle

@@ -0,0 +1,6 @@
+dependencies {
+    compile project(':embulk-core')
+    compile 'com.amazonaws:aws-java-sdk:1.5.2'
+
+    testCompile project(':embulk-core').sourceSets.test.output
+}

data/embulk-standards/pom.xml

@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.embulk</groupId>
+        <artifactId>embulk-parent</artifactId>
+        <version>0.1.0-SNAPSHOT</version>
+    </parent>
+
+    <artifactId>embulk-standards</artifactId>
+    <name>embulk-standards</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.embulk</groupId>
+            <artifactId>embulk-core</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.embulk</groupId>
+            <artifactId>embulk-core</artifactId>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.google.inject</groupId>
+            <artifactId>guice</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>javax.validation</groupId>
+            <artifactId>validation-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.fasterxml.jackson.core</groupId>
+            <artifactId>jackson-databind</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk</artifactId>
+            <version>1.5.2</version>
+        </dependency>
+
+        <dependency>
+            <groupId>junit</groupId>
+            <artifactId>junit</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.mockito</groupId>
+            <artifactId>mockito-core</artifactId>
+        </dependency>
+    </dependencies>
+</project>

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -0,0 +1,158 @@
+package org.embulk.standards;
+
+import com.google.common.collect.ImmutableBiMap;
+import com.google.common.collect.ImmutableMap;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.FormatterPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.util.LineEncoder;
+
+import java.util.Map;
+
+public class CsvFormatterPlugin
+        implements FormatterPlugin
+{
+    public interface PluginTask
+            extends LineEncoder.EncoderTask, TimestampFormatter.FormatterTask
+    {
+        @Config("header_line")
+        @ConfigDefault("true")
+        public boolean getHeaderLine();
+    }
+
+    @Override
+    public void transaction(ConfigSource config, Schema schema,
+            FormatterPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump());
+    }
+
+    private Map<Integer, TimestampFormatter> newTimestampFormatters(
+            TimestampFormatter.FormatterTask task, Schema schema)
+    {
+        ImmutableMap.Builder<Integer, TimestampFormatter> builder = new ImmutableBiMap.Builder<>();
+        for (Column column : schema.getColumns()) {
+            if (column.getType() instanceof TimestampType) {
+                TimestampType tt = (TimestampType) column.getType();
+                builder.put(column.getIndex(), new TimestampFormatter(tt.getFormat(), task));
+            }
+        }
+        return builder.build();
+    }
+
+    @Override
+    public PageOutput open(TaskSource taskSource, final Schema schema,
+            FileOutput output)
+    {
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
+        final LineEncoder encoder = new LineEncoder(output, task);
+        final Map<Integer, TimestampFormatter> timestampFormatters =
+                newTimestampFormatters(task, schema);
+
+        // create a file
+        encoder.nextFile();
+
+        // write header
+        if (task.getHeaderLine()) {
+            writeHeader(schema, encoder);
+        }
+
+        return new PageOutput() {
+            private final PageReader pageReader = new PageReader(schema);
+
+            public void add(Page page)
+            {
+                pageReader.setPage(page);
+                while (pageReader.nextRecord()) {
+                    schema.visitColumns(new SchemaVisitor() {
+                        public void booleanColumn(Column column)
+                        {
+                            addDelimiter(column);
+                            if (!pageReader.isNull(column)) {
+                                encoder.addText(Boolean.toString(pageReader.getBoolean(column)));
+                            }
+                        }
+
+                        public void longColumn(Column column)
+                        {
+                            addDelimiter(column);
+                            if (!pageReader.isNull(column)) {
+                                encoder.addText(Long.toString(pageReader.getLong(column)));
+                            }
+                        }
+
+                        public void doubleColumn(Column column)
+                        {
+                            addDelimiter(column);
+                            if (!pageReader.isNull(column)) {
+                                encoder.addText(Double.toString(pageReader.getDouble(column)));
+                            }
+                        }
+
+                        public void stringColumn(Column column)
+                        {
+                            addDelimiter(column);
+                            if (!pageReader.isNull(column)) {
+                                // TODO escape and quoting
+                                encoder.addText(pageReader.getString(column));
+                            }
+                        }
+
+                        public void timestampColumn(Column column)
+                        {
+                            addDelimiter(column);
+                            if (!pageReader.isNull(column)) {
+                                Timestamp value = pageReader.getTimestamp(column);
+                                encoder.addText(timestampFormatters.get(column.getIndex()).format(value));
+                            }
+                        }
+
+                        private void addDelimiter(Column column)
+                        {
+                            if (column.getIndex() != 0) {
+                                encoder.addText(",");
+                            }
+                        }
+                    });
+
+                    encoder.addNewLine();
+                }
+            }
+
+            public void finish()
+            {
+                encoder.finish();
+            }
+
+            public void close()
+            {
+                encoder.close();
+            }
+        };
+    }
+
+    private void writeHeader(Schema schema, LineEncoder encoder)
+    {
+        for (Column column : schema.getColumns()) {
+            if (column.getIndex() != 0) {
+                encoder.addText(",");
+            }
+            encoder.addText(column.getName());
+        }
+        encoder.addNewLine();
+    }
+}

data/embulk-standards/src/main/java/org/embulk/standards/CsvParserPlugin.java

@@ -0,0 +1,233 @@
+package org.embulk.standards;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.base.Optional;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParseException;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.SchemaVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.ParserPlugin;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.util.LineDecoder;
+import org.slf4j.Logger;
+
+import java.util.Map;
+
+public class CsvParserPlugin
+        implements ParserPlugin
+{
+    public interface PluginTask
+            extends Task, LineDecoder.DecoderTask, TimestampParser.ParserTask
+    {
+        @Config("columns")
+        public SchemaConfig getSchemaConfig();
+
+        @Config("header_line") // how to set default value?? TODO @Default("true")
+        @ConfigDefault("false")
+        public boolean getHeaderLine();
+
+        @Config("delimiter")
+        @ConfigDefault("\",\"")
+        public char getDelimiterChar();
+
+        @Config("quote")
+        @ConfigDefault("\"\\\"\"")
+        public char getQuoteChar();
+
+        @Config("escape")
+        @ConfigDefault("\"\\\\\"")
+        public char getEscapeChar();
+
+        // Null value handling: if the CsvParser found 'non-quoted empty string's,
+        // it replaces them to string that users specified like "\N", "NULL".
+        @Config("null_string")
+        @ConfigDefault("null")
+        public Optional<String> getNullString();
+
+        @Config("trim_if_not_quoted")
+        @ConfigDefault("false")
+        public boolean getTrimIfNotQuoted();
+
+        @Config("max_quoted_size_limit")
+        @ConfigDefault("131072") //128kB
+        public long getMaxQuotedSizeLimit();
+    }
+
+    private final Logger log;
+
+    public CsvParserPlugin()
+    {
+        log = Exec.getLogger(CsvParserPlugin.class);
+    }
+
+    @Override
+    public void transaction(ConfigSource config, ParserPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        control.run(task.dump(), task.getSchemaConfig().toSchema());
+    }
+
+    private Map<Integer, TimestampParser> newTimestampParsers(
+            TimestampParser.ParserTask task, Schema schema)
+    {
+        ImmutableMap.Builder<Integer, TimestampParser> builder = new ImmutableMap.Builder<>();
+        for (Column column : schema.getColumns()) {
+            if (column.getType() instanceof TimestampType) {
+                TimestampType tt = (TimestampType) column.getType();
+                builder.put(column.getIndex(), new TimestampParser(tt.getFormat(), task));
+            }
+        }
+        return builder.build();
+    }
+
+    @Override
+    public void run(TaskSource taskSource, final Schema schema,
+            FileInput input, PageOutput output)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final Map<Integer, TimestampParser> timestampFormatters = newTimestampParsers(task, schema);
+        final CsvTokenizer tokenizer = new CsvTokenizer(new LineDecoder(input, task), task);
+        final String nullStringOrNull = task.getNullString().orNull();
+        boolean skipHeaderLine = task.getHeaderLine();
+
+        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
+            while (tokenizer.nextFile()) {
+                if (skipHeaderLine) {
+                    // skip the first line
+                    if (tokenizer.nextRecord()) {
+                        for (int i=0; i < schema.getColumnCount(); i++) {
+                            tokenizer.nextColumn();  // TODO check return value?
+                        }
+                    }
+                }
+
+                while (true) {
+                    try {
+                        if (!tokenizer.nextRecord()) {
+                            break;
+                        }
+
+                        schema.visitColumns(new SchemaVisitor() {
+                            public void booleanColumn(Column column)
+                            {
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    pageBuilder.setBoolean(column, Boolean.parseBoolean(v));
+                                }
+                            }
+
+                            public void longColumn(Column column)
+                            {
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setLong(column, Long.parseLong(v));
+                                    } catch (NumberFormatException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            public void doubleColumn(Column column)
+                            {
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setDouble(column, Double.parseDouble(v));
+                                    } catch (NumberFormatException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+
+                            public void stringColumn(Column column)
+                            {
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    pageBuilder.setString(column, v);
+                                }
+                            }
+
+                            public void timestampColumn(Column column)
+                            {
+                                String v = nextColumn(schema, tokenizer, nullStringOrNull);
+                                if (v == null) {
+                                    pageBuilder.setNull(column);
+                                } else {
+                                    try {
+                                        pageBuilder.setTimestamp(column, (timestampFormatters.get(column.getIndex()).parse(v)));
+                                    } catch (TimestampParseException e) {
+                                        // TODO support default value
+                                        throw new CsvRecordValidateException(e);
+                                    }
+                                }
+                            }
+                        });
+                        pageBuilder.addRecord();
+
+                    } catch (Exception e) {
+                        // TODO logging
+                        long lineNumber = tokenizer.getCurrentLineNumber();
+                        String skippedLine = tokenizer.skipCurrentLine();
+                        log.warn(String.format("Skipped (line %d): %s", lineNumber, skippedLine), e);
+                        //exec.notice().skippedLine(skippedLine);
+                    }
+                }
+            }
+
+            pageBuilder.finish();
+        }
+    }
+
+    private static String nextColumn(Schema schema, CsvTokenizer tokenizer, String nullStringOrNull)
+    {
+        String v = tokenizer.nextColumn();
+        if (v == null) {
+            throw new RuntimeException(String.format("Expected %d columns but line %d has fewer number of columns",
+                        schema.getColumnCount(), tokenizer.getCurrentLineNumber()));
+        }
+
+        if (!v.isEmpty()) {
+            if (v.equals(nullStringOrNull)) {
+                return null;
+            }
+            return v;
+        } else if (tokenizer.wasQuotedColumn()) {
+            return "";
+        } else {
+            return null;
+        }
+    }
+
+    static class CsvRecordValidateException
+            extends RuntimeException
+    {
+        CsvRecordValidateException(Throwable cause)
+        {
+            super(cause);
+        }
+    }
+}