embulk 0.8.38-java → 0.10.27-java

data/embulk-standards/build.gradle

@@ -1,7 +0,0 @@
-dependencies {
-    compile project(':embulk-core')
-    compile 'org.apache.commons:commons-compress:1.10'
-
-    testCompile project(':embulk-core').sourceSets.test.output
-    testCompile project(':embulk-test')
-}

data/embulk-standards/src/main/java/org/embulk/standards/Bzip2FileDecoderPlugin.java

@@ -1,55 +0,0 @@
-package org.embulk.standards;
-
-import java.io.InputStream;
-import java.io.IOException;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigInject;
-import org.embulk.spi.DecoderPlugin;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.util.FileInputInputStream;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorInputStream;
-
-public class Bzip2FileDecoderPlugin
-        implements DecoderPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
-    @Override
-    public void transaction(ConfigSource config, DecoderPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        control.run(task.dump());
-    }
-
-    @Override
-    public FileInput open(TaskSource taskSource, FileInput fileInput)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        final FileInputInputStream files = new FileInputInputStream(fileInput);
-        return new InputStreamFileInput(
-                task.getBufferAllocator(),
-                new InputStreamFileInput.Provider() {
-                    public InputStream openNext() throws IOException
-                    {
-                        if (!files.nextFile()) {
-                            return null;
-                        }
-                        return new BZip2CompressorInputStream(files, true);
-                    }
-
-                    public void close() throws IOException
-                    {
-                        files.close();
-                    }
-                });
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/Bzip2FileEncoderPlugin.java

@@ -1,67 +0,0 @@
-package org.embulk.standards;
-
-import java.io.OutputStream;
-import java.io.IOException;
-import javax.validation.constraints.Min;
-import javax.validation.constraints.Max;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.EncoderPlugin;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.util.FileOutputOutputStream;
-import org.embulk.spi.util.OutputStreamFileOutput;
-import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
-
-public class Bzip2FileEncoderPlugin
-        implements EncoderPlugin
-{
-    public interface PluginTask
-            extends Task
-    {
-        @Config("level")
-        @ConfigDefault("9")
-        @Min(1)
-        @Max(9)
-        int getLevel();
-
-        @ConfigInject
-        BufferAllocator getBufferAllocator();
-    }
-
-    public void transaction(ConfigSource config, EncoderPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        control.run(task.dump());
-    }
-
-    @Override
-    public FileOutput open(TaskSource taskSource, final FileOutput fileOutput)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH);
-
-        return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() {
-            public OutputStream openNext() throws IOException
-            {
-                output.nextFile();
-                return new BZip2CompressorOutputStream(output, task.getLevel());
-            }
-
-            public void finish() throws IOException
-            {
-                fileOutput.finish();
-            }
-
-            public void close() throws IOException
-            {
-                fileOutput.close();
-            }
-        });
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/ConfigInputPlugin.java

@@ -1,170 +0,0 @@
-package org.embulk.standards;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import java.util.List;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.DataException;
-import org.embulk.spi.Exec;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.json.JsonParser;
-import org.embulk.spi.json.JsonParseException;
-import org.embulk.spi.time.TimestampParseException;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.util.Timestamps;
-
-public class ConfigInputPlugin
-        implements InputPlugin
-{
-    private interface PluginTask
-            extends Task, TimestampParser.Task
-    {
-        @Config("columns")
-        SchemaConfig getSchemaConfig();
-
-        @Config("values")
-        List<List<List<JsonNode>>> getValues();
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            InputPlugin.Control control)
-    {
-        final PluginTask task = config.loadConfig(PluginTask.class);
-        final Schema schema = task.getSchemaConfig().toSchema();
-        final List<List<List<JsonNode>>> values = task.getValues();
-        final int taskCount = values.size();
-
-        return resume(task.dump(), schema, taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control)
-    {
-        control.run(taskSource, schema, taskCount);
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    {
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource,
-            Schema schema, int taskIndex,
-            PageOutput output)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-        final List<List<JsonNode>> taskValues = task.getValues().get(taskIndex);
-        final TimestampParser[] timestampParsers = Timestamps.newTimestampColumnParsers(task, task.getSchemaConfig());
-        final JsonParser jsonParser = new JsonParser();
-
-        try (final PageBuilder pageBuilder = new PageBuilder(Exec.getBufferAllocator(), schema, output)) {
-            for (final List<JsonNode> rowValues : taskValues) {
-                schema.visitColumns(new ColumnVisitor() {
-                        public void booleanColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                pageBuilder.setBoolean(column, value.asBoolean());
-                            }
-                        }
-
-                        public void longColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                pageBuilder.setLong(column, value.asLong());
-                            }
-                        }
-
-                        public void doubleColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                pageBuilder.setDouble(column, value.asDouble());
-                            }
-                        }
-
-                        public void stringColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                pageBuilder.setString(column, value.asText());
-                            }
-                        }
-
-                        public void timestampColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                try {
-                                    pageBuilder.setTimestamp(column,
-                                                             timestampParsers[column.getIndex()].parse(value.asText()));
-                                }
-                                catch (TimestampParseException ex) {
-                                    throw new DataException(ex);
-                                }
-                            }
-                        }
-
-                        public void jsonColumn(Column column)
-                        {
-                            final JsonNode value = rowValues.get(column.getIndex());
-                            if (value == null || value.isNull()) {
-                                pageBuilder.setNull(column);
-                            }
-                            else {
-                                try {
-                                    pageBuilder.setJson(column, jsonParser.parse(value.toString()));
-                                }
-                                catch (JsonParseException ex) {
-                                    throw new DataException(ex);
-                                }
-                            }
-                        }
-                    });
-                pageBuilder.addRecord();
-            }
-            pageBuilder.finish();
-        }
-
-        return Exec.newTaskReport();
-    }
-
-    @Override
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return Exec.newConfigDiff();
-    }
-}

data/embulk-standards/src/main/java/org/embulk/standards/CsvFormatterPlugin.java

@@ -1,295 +0,0 @@
-package org.embulk.standards;
-
-import com.google.common.base.Optional;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.Column;
-import org.embulk.spi.Schema;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.FormatterPlugin;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.util.LineEncoder;
-import org.embulk.spi.util.Timestamps;
-import org.embulk.spi.util.Newline;
-import org.msgpack.value.Value;
-import java.util.Map;
-
-public class CsvFormatterPlugin
-        implements FormatterPlugin
-{
-    public enum QuotePolicy
-    {
-        ALL("ALL"),
-        MINIMAL("MINIMAL"),
-        NONE("NONE");
-
-        private final String string;
-
-        private QuotePolicy(String string)
-        {
-            this.string = string;
-        }
-
-        public String getString()
-        {
-            return string;
-        }
-    }
-
-    public interface PluginTask
-            extends Task, LineEncoder.EncoderTask, TimestampFormatter.Task
-    {
-        @Config("header_line")
-        @ConfigDefault("true")
-        boolean getHeaderLine();
-
-        @Config("delimiter")
-        @ConfigDefault("\",\"")
-        char getDelimiterChar();
-
-        @Config("quote")
-        @ConfigDefault("\"\\\"\"")
-        char getQuoteChar();
-
-        @Config("quote_policy")
-        @ConfigDefault("\"MINIMAL\"")
-        QuotePolicy getQuotePolicy();
-
-        @Config("escape")
-        @ConfigDefault("null")
-        Optional<Character> getEscapeChar();
-
-        @Config("null_string")
-        @ConfigDefault("\"\"")
-        String getNullString();
-
-        @Config("newline_in_field")
-        @ConfigDefault("\"LF\"")
-        Newline getNewlineInField();
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        Map<String, TimestampColumnOption> getColumnOptions();
-    }
-
-    public interface TimestampColumnOption
-            extends Task, TimestampFormatter.TimestampColumnOption
-    { }
-
-    @Override
-    public void transaction(ConfigSource config, Schema schema,
-            FormatterPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // validate column_options
-        for (String columnName : task.getColumnOptions().keySet()) {
-            schema.lookupColumn(columnName);  // throws SchemaConfigException
-        }
-
-        control.run(task.dump());
-    }
-
-    @Override
-    public PageOutput open(TaskSource taskSource, final Schema schema,
-            FileOutput output)
-    {
-        final PluginTask task = taskSource.loadTask(PluginTask.class);
-        final LineEncoder encoder = new LineEncoder(output, task);
-        final TimestampFormatter[] timestampFormatters = Timestamps.newTimestampColumnFormatters(task, schema, task.getColumnOptions());
-        final char delimiter = task.getDelimiterChar();
-        final QuotePolicy quotePolicy = task.getQuotePolicy();
-        final char quote = task.getQuoteChar() != '\0' ? task.getQuoteChar() : '"';
-        final char escape = task.getEscapeChar().or(quotePolicy == QuotePolicy.NONE ? '\\' : quote);
-        final String newlineInField = task.getNewlineInField().getString();
-        final String nullString = task.getNullString();
-
-        // create a file
-        encoder.nextFile();
-
-        // write header
-        if (task.getHeaderLine()) {
-            writeHeader(schema, encoder, delimiter, quotePolicy, quote, escape, newlineInField, nullString);
-        }
-
-        return new PageOutput() {
-            private final PageReader pageReader = new PageReader(schema);
-            private final String delimiterString = String.valueOf(delimiter);
-
-            public void add(Page page)
-            {
-                pageReader.setPage(page);
-                while (pageReader.nextRecord()) {
-                    schema.visitColumns(new ColumnVisitor() {
-                        public void booleanColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                addValue(Boolean.toString(pageReader.getBoolean(column)));
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        public void longColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                addValue(Long.toString(pageReader.getLong(column)));
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        public void doubleColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                addValue(Double.toString(pageReader.getDouble(column)));
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        public void stringColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                addValue(pageReader.getString(column));
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        public void timestampColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                Timestamp value = pageReader.getTimestamp(column);
-                                addValue(timestampFormatters[column.getIndex()].format(value));
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        public void jsonColumn(Column column)
-                        {
-                            addDelimiter(column);
-                            if (!pageReader.isNull(column)) {
-                                Value value = pageReader.getJson(column);
-                                addValue(value.toJson());
-                            } else {
-                                addNullString();
-                            }
-                        }
-
-                        private void addDelimiter(Column column)
-                        {
-                            if (column.getIndex() != 0) {
-                                encoder.addText(delimiterString);
-                            }
-                        }
-
-                        private void addValue(String v)
-                        {
-                            encoder.addText(setEscapeAndQuoteValue(v, delimiter, quotePolicy, quote, escape, newlineInField, nullString));
-                        }
-
-                        private void addNullString()
-                        {
-                            encoder.addText(nullString);
-                        }
-                    });
-                    encoder.addNewLine();
-                }
-            }
-
-            public void finish()
-            {
-                encoder.finish();
-            }
-
-            public void close()
-            {
-                encoder.close();
-            }
-        };
-    }
-
-    private void writeHeader(Schema schema, LineEncoder encoder, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
-    {
-        String delimiterString = String.valueOf(delimiter);
-        for (Column column : schema.getColumns()) {
-            if (column.getIndex() != 0) {
-                encoder.addText(delimiterString);
-            }
-            encoder.addText(setEscapeAndQuoteValue(column.getName(), delimiter, policy, quote, escape, newline, nullString));
-        }
-        encoder.addNewLine();
-    }
-
-    private String setEscapeAndQuoteValue(String v, char delimiter, QuotePolicy policy, char quote, char escape, String newline, String nullString)
-    {
-        StringBuilder escapedValue = new StringBuilder();
-        char previousChar = ' ';
-
-        boolean isRequireQuote = (policy == QuotePolicy.ALL || policy == QuotePolicy.MINIMAL && v.equals(nullString)) ? true : false;
-
-        for (int i = 0; i < v.length(); i++) {
-            char c = v.charAt(i);
-
-            if (policy != QuotePolicy.NONE && c == quote) {
-                escapedValue.append(escape);
-                escapedValue.append(c);
-                isRequireQuote = true;
-            } else if (c == '\r') {
-                if (policy == QuotePolicy.NONE) {
-                    escapedValue.append(escape);
-                }
-                escapedValue.append(newline);
-                isRequireQuote = true;
-            } else if (c == '\n') {
-                if (previousChar != '\r') {
-                    if (policy == QuotePolicy.NONE) {
-                        escapedValue.append(escape);
-                    }
-                    escapedValue.append(newline);
-                    isRequireQuote = true;
-                }
-            } else if (c == delimiter) {
-                if (policy == QuotePolicy.NONE) {
-                    escapedValue.append(escape);
-                }
-                escapedValue.append(c);
-                isRequireQuote = true;
-            } else {
-                escapedValue.append(c);
-            }
-            previousChar = c;
-        }
-
-        if (policy != QuotePolicy.NONE && isRequireQuote) {
-            return setQuoteValue(escapedValue.toString(), quote);
-        } else {
-            return escapedValue.toString();
-        }
-    }
-
-    private String setQuoteValue(String v, char quote)
-    {
-        StringBuilder sb = new StringBuilder();
-        sb.append(quote);
-        sb.append(v);
-        sb.append(quote);
-
-        return sb.toString();
-    }
-}