embulk 0.8.35-java → 0.10.24-java

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnNotFoundException.java

@@ -1,10 +0,0 @@
-package org.embulk.spi.util;
-
-public class DynamicColumnNotFoundException
-        extends RuntimeException
-{
-    public DynamicColumnNotFoundException(String message)
-    {
-        super(message);
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetter.java

@@ -1,21 +0,0 @@
-package org.embulk.spi.util;
-
-import org.embulk.spi.time.Timestamp;
-import org.msgpack.value.Value;
-
-public interface DynamicColumnSetter
-{
-    void setNull();
-
-    void set(boolean value);
-
-    void set(long value);
-
-    void set(double value);
-
-    void set(String value);
-
-    void set(Timestamp value);
-
-    void set(Value value);
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicColumnSetterFactory.java

@@ -1,100 +0,0 @@
-package org.embulk.spi.util;
-
-import org.joda.time.DateTimeZone;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.type.Type;
-import org.embulk.spi.type.BooleanType;
-import org.embulk.spi.type.LongType;
-import org.embulk.spi.type.DoubleType;
-import org.embulk.spi.type.StringType;
-import org.embulk.spi.type.TimestampType;
-import org.embulk.spi.type.JsonType;
-import org.embulk.spi.util.dynamic.BooleanColumnSetter;
-import org.embulk.spi.util.dynamic.LongColumnSetter;
-import org.embulk.spi.util.dynamic.DoubleColumnSetter;
-import org.embulk.spi.util.dynamic.StringColumnSetter;
-import org.embulk.spi.util.dynamic.TimestampColumnSetter;
-import org.embulk.spi.util.dynamic.JsonColumnSetter;
-import org.embulk.spi.util.dynamic.DefaultValueSetter;
-import org.embulk.spi.util.dynamic.NullDefaultValueSetter;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.time.TimestampFormat;
-import org.embulk.spi.Column;
-import org.embulk.spi.PageBuilder;
-import org.embulk.config.ConfigException;
-
-public class DynamicColumnSetterFactory
-{
-    private final DefaultValueSetter defaultValue;
-    private final DynamicPageBuilder.BuilderTask task;
-
-    DynamicColumnSetterFactory(
-            DynamicPageBuilder.BuilderTask task,
-            DefaultValueSetter defaultValue)
-    {
-        this.defaultValue = defaultValue;
-        this.task = task;
-    }
-
-    public static DefaultValueSetter nullDefaultValue()
-    {
-        return new NullDefaultValueSetter();
-    }
-
-    public DynamicColumnSetter newColumnSetter(PageBuilder pageBuilder, Column column)
-    {
-        Type type = column.getType();
-        if (type instanceof BooleanType) {
-            return new BooleanColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof LongType) {
-            return new LongColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof DoubleType) {
-            return new DoubleColumnSetter(pageBuilder, column, defaultValue);
-        } else if (type instanceof StringType) {
-            TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
-            return new StringColumnSetter(pageBuilder, column, defaultValue, formatter);
-        } else if (type instanceof TimestampType) {
-            // TODO use flexible time format like Ruby's Time.parse
-            TimestampParser parser = new TimestampParser(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
-            return new TimestampColumnSetter(pageBuilder, column, defaultValue, parser);
-        } else if (type instanceof JsonType) {
-            TimestampFormatter formatter = new TimestampFormatter(
-                    getTimestampFormat(column).getFormat(), getTimeZone(column));
-            return new JsonColumnSetter(pageBuilder, column, defaultValue, formatter);
-        }
-        throw new ConfigException("Unknown column type: "+type);
-    }
-
-    private TimestampFormat getTimestampFormat(Column column)
-    {
-        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
-        if (option != null) {
-            return option.getTimestampFormat();
-        } else {
-            return new TimestampFormat("%Y-%m-%d %H:%M:%S.%6N");
-        }
-    }
-
-    private DateTimeZone getTimeZone(Column column)
-    {
-        DynamicPageBuilder.ColumnOption option = getColumnOption(column);
-        if (option != null) {
-            return option.getTimeZone().or(task.getDefaultTimeZone());
-        } else {
-            return task.getDefaultTimeZone();
-        }
-    }
-
-    private DynamicPageBuilder.ColumnOption getColumnOption(Column column)
-    {
-        ConfigSource option = task.getColumnOptions().get(column.getName());
-        if (option != null) {
-            return option.loadConfig(DynamicPageBuilder.ColumnOption.class);
-        } else {
-            return null;
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -1,162 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import java.util.Map;
-import com.google.common.base.Optional;
-import org.joda.time.DateTimeZone;
-import org.jruby.embed.ScriptingContainer;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Column;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.time.TimestampFormat;
-import org.embulk.spi.util.dynamic.SkipColumnSetter;
-
-public class DynamicPageBuilder
-        implements AutoCloseable
-{
-    private final PageBuilder pageBuilder;
-    private final Schema schema;
-    private final DynamicColumnSetter[] setters;
-    private final Map<String, DynamicColumnSetter> columnLookup;
-
-    public static interface BuilderTask
-            extends Task
-    {
-        @Config("default_timezone")
-        @ConfigDefault("\"UTC\"")
-        public DateTimeZone getDefaultTimeZone();
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        public Map<String, ConfigSource> getColumnOptions();
-
-        // required by TimestampFormatter
-        @ConfigInject
-        @Deprecated
-        public ScriptingContainer getJRuby();
-    }
-
-    public static interface ColumnOption
-            extends Task
-    {
-        @Config("timestamp_format")
-        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"")
-        public TimestampFormat getTimestampFormat();
-
-        @Config("timezone")
-        @ConfigDefault("null")
-        public Optional<DateTimeZone> getTimeZone();
-    }
-
-    public DynamicPageBuilder(BuilderTask task,
-            BufferAllocator allocator, Schema schema, PageOutput output)
-    {
-        this.pageBuilder = new PageBuilder(allocator, schema, output);
-        this.schema = schema;
-
-        // TODO configurable default value
-        DynamicColumnSetterFactory factory = new DynamicColumnSetterFactory(task,
-                DynamicColumnSetterFactory.nullDefaultValue());
-
-        ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder();
-        ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder();
-        for (Column c : schema.getColumns()) {
-            DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c);
-            setters.add(setter);
-            lookup.put(c.getName(), setter);
-        }
-        this.setters = setters.build().toArray(new DynamicColumnSetter[0]);
-        this.columnLookup = lookup.build();
-    }
-
-    public List<Column> getColumns()
-    {
-        return schema.getColumns();
-    }
-
-    public DynamicColumnSetter column(Column c)
-    {
-        return setters[c.getIndex()];
-    }
-
-    public DynamicColumnSetter column(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            throw new DynamicColumnNotFoundException("Column index '"+index+"' is not exist");
-        }
-        return setters[index];
-    }
-
-    public DynamicColumnSetter lookupColumn(String columnName)
-    {
-        DynamicColumnSetter setter = columnLookup.get(columnName);
-        if (setter == null) {
-            throw new DynamicColumnNotFoundException("Column '"+columnName+"' is not exist");
-        }
-        return setter;
-    }
-
-    public DynamicColumnSetter columnOrSkip(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            return SkipColumnSetter.get();
-        }
-        return setters[index];
-    }
-
-    public DynamicColumnSetter columnOrSkip(String columnName)
-    {
-        DynamicColumnSetter setter = columnLookup.get(columnName);
-        if (setter == null) {
-            return SkipColumnSetter.get();
-        }
-        return setter;
-    }
-
-    // for jruby
-    protected DynamicColumnSetter columnOrNull(int index)
-    {
-        if (index < 0 || setters.length <= index) {
-            return null;
-        }
-        return setters[index];
-    }
-
-    // for jruby
-    protected DynamicColumnSetter columnOrNull(String columnName)
-    {
-        return columnLookup.get(columnName);
-    }
-
-    public void addRecord()
-    {
-        pageBuilder.addRecord();
-    }
-
-    public void flush()
-    {
-        pageBuilder.flush();
-    }
-
-    public void finish()
-    {
-        pageBuilder.finish();
-    }
-
-    @Override
-    public void close()
-    {
-        pageBuilder.close();
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java

@@ -1,81 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.plugin.PluginType;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.EncoderPlugin;
-
-public abstract class Encoders
-{
-    private Encoders() { }
-
-    public static List<EncoderPlugin> newEncoderPlugins(ExecSession exec, List<ConfigSource> configs)
-    {
-        ImmutableList.Builder<EncoderPlugin> builder = ImmutableList.builder();
-        for (ConfigSource config : configs) {
-            builder.add(exec.newPlugin(EncoderPlugin.class, config.get(PluginType.class, "type")));
-        }
-        return builder.build();
-    }
-
-    public interface Control
-    {
-        public void run(List<TaskSource> taskSources);
-    }
-
-    public static void transaction(List<EncoderPlugin> plugins, List<ConfigSource> configs,
-            Encoders.Control control)
-    {
-        new RecursiveControl(plugins, configs, control).transaction();
-    }
-
-    public static FileOutput open(List<EncoderPlugin> plugins, List<TaskSource> taskSources,
-            FileOutput output)
-    {
-        FileOutput out = output;
-        int pos = 0;
-        while (pos < plugins.size()) {
-            out = plugins.get(pos).open(taskSources.get(pos), out);
-            pos++;
-        }
-        return out;
-    }
-
-    private static class RecursiveControl
-    {
-        private final List<EncoderPlugin> plugins;
-        private final List<ConfigSource> configs;
-        private final Encoders.Control finalControl;
-        private final ImmutableList.Builder<TaskSource> taskSources;
-        private int pos;
-
-        RecursiveControl(List<EncoderPlugin> plugins, List<ConfigSource> configs,
-                Encoders.Control finalControl)
-        {
-            this.plugins = plugins;
-            this.configs = configs;
-            this.finalControl = finalControl;
-            this.taskSources = ImmutableList.builder();
-        }
-
-        public void transaction()
-        {
-            if (pos < plugins.size()) {
-                plugins.get(pos).transaction(configs.get(pos), new EncoderPlugin.Control() {
-                    public void run(TaskSource taskSource)
-                    {
-                        taskSources.add(taskSource);
-                        pos++;
-                        transaction();
-                    }
-                });
-            } else {
-                finalControl.run(taskSources.build());
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java

@@ -1,93 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.ProcessState;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.FilterPlugin;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.ProcessTask;
-import org.embulk.spi.AbortTransactionResource;
-import org.embulk.spi.CloseResource;
-import org.embulk.plugin.compat.PluginWrappers;
-
-public abstract class Executors
-{
-    private Executors() { }
-
-    public interface ProcessStateCallback
-    {
-        public void started();
-
-        public void inputCommitted(TaskReport report);
-
-        public void outputCommitted(TaskReport report);
-    }
-
-    public static void process(ExecSession exec,
-            ProcessTask task, int taskIndex,
-            ProcessStateCallback callback)
-    {
-        InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
-        List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
-        OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
-
-        // TODO assert task.getExecutorSchema().equals task.getOutputSchema()
-
-        process(exec, taskIndex,
-                inputPlugin, task.getInputSchema(), task.getInputTaskSource(),
-                filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources(),
-                outputPlugin, task.getOutputSchema(), task.getOutputTaskSource(),
-                callback);
-    }
-
-    public static void process(ExecSession exec, int taskIndex,
-            InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource,
-            List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources,
-            OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource,
-            ProcessStateCallback callback)
-    {
-        TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
-            outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
-
-        callback.started();
-        // here needs to use try-with-resource to add exception happend at close() or abort()
-        // to suppressed exception. otherwise exception happend at close() or abort() overwrites
-        // essential exception.
-        try (CloseResource closer = new CloseResource(tran)) {
-            try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
-                PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
-                closer.closeThis(filtered);
-
-                TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
-
-                if (inputTaskReport == null) {
-                    inputTaskReport = exec.newTaskReport();
-                }
-                callback.inputCommitted(inputTaskReport);
-
-                TaskReport outputTaskReport = tran.commit();
-                aborter.dontAbort();
-                if (outputTaskReport == null) {
-                    outputTaskReport = exec.newTaskReport();
-                }
-                callback.outputCommitted(outputTaskReport);  // TODO check output.finish() is called. wrap or abstract
-            }
-        }
-    }
-
-    public static Schema getInputSchema(List<Schema> schemas)
-    {
-        return schemas.get(0);
-    }
-
-    public static Schema getOutputSchema(List<Schema> schemas)
-    {
-        return schemas.get(schemas.size() - 1);
-    }
-}