embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/util/DynamicPageBuilder.java

@@ -0,0 +1,161 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.Map;
+import com.google.common.base.Optional;
+import org.joda.time.DateTimeZone;
+import org.jruby.embed.ScriptingContainer;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampFormat;
+import org.embulk.spi.util.dynamic.SkipColumnSetter;
+
+public class DynamicPageBuilder
+        implements AutoCloseable
+{
+    private final PageBuilder pageBuilder;
+    private final Schema schema;
+    private final DynamicColumnSetter[] setters;
+    private final Map<String, DynamicColumnSetter> columnLookup;
+
+    public static interface BuilderTask
+            extends Task
+    {
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        public Map<String, ConfigSource> getColumnOptions();
+
+        // required by TimestampFormatter
+        @ConfigInject
+        public ScriptingContainer getJRuby();
+    }
+
+    public static interface ColumnOption
+            extends Task
+    {
+        @Config("timestamp_format")
+        @ConfigDefault("\"%Y-%m-%d %H:%M:%S.%6N\"")
+        public TimestampFormat getTimestampFormat();
+
+        @Config("timezone")
+        @ConfigDefault("null")
+        public Optional<DateTimeZone> getTimeZone();
+    }
+
+    public DynamicPageBuilder(BuilderTask task,
+            BufferAllocator allocator, Schema schema, PageOutput output)
+    {
+        this.pageBuilder = new PageBuilder(allocator, schema, output);
+        this.schema = schema;
+
+        // TODO configurable default value
+        DynamicColumnSetterFactory factory = new DynamicColumnSetterFactory(task,
+                DynamicColumnSetterFactory.nullDefaultValue());
+
+        ImmutableList.Builder<DynamicColumnSetter> setters = ImmutableList.builder();
+        ImmutableMap.Builder<String, DynamicColumnSetter> lookup = ImmutableMap.builder();
+        for (Column c : schema.getColumns()) {
+            DynamicColumnSetter setter = factory.newColumnSetter(pageBuilder, c);
+            setters.add(setter);
+            lookup.put(c.getName(), setter);
+        }
+        this.setters = setters.build().toArray(new DynamicColumnSetter[0]);
+        this.columnLookup = lookup.build();
+    }
+
+    public List<Column> getColumns()
+    {
+        return schema.getColumns();
+    }
+
+    public DynamicColumnSetter column(Column c)
+    {
+        return setters[c.getIndex()];
+    }
+
+    public DynamicColumnSetter column(int index)
+    {
+        if (index < 0 || setters.length <= index) {
+            throw new DynamicColumnNotFoundException("Column index '"+index+"' is not exist");
+        }
+        return setters[index];
+    }
+
+    public DynamicColumnSetter lookupColumn(String columnName)
+    {
+        DynamicColumnSetter setter = columnLookup.get(columnName);
+        if (setter == null) {
+            throw new DynamicColumnNotFoundException("Column '"+columnName+"' is not exist");
+        }
+        return setter;
+    }
+
+    public DynamicColumnSetter columnOrSkip(int index)
+    {
+        if (index < 0 || setters.length <= index) {
+            return SkipColumnSetter.get();
+        }
+        return setters[index];
+    }
+
+    public DynamicColumnSetter columnOrSkip(String columnName)
+    {
+        DynamicColumnSetter setter = columnLookup.get(columnName);
+        if (setter == null) {
+            return SkipColumnSetter.get();
+        }
+        return setter;
+    }
+
+    // for jruby
+    protected DynamicColumnSetter columnOrNull(int index)
+    {
+        if (index < 0 || setters.length <= index) {
+            return null;
+        }
+        return setters[index];
+    }
+
+    // for jruby
+    protected DynamicColumnSetter columnOrNull(String columnName)
+    {
+        return columnLookup.get(columnName);
+    }
+
+    public void addRecord()
+    {
+        pageBuilder.addRecord();
+    }
+
+    public void flush()
+    {
+        pageBuilder.flush();
+    }
+
+    public void finish()
+    {
+        pageBuilder.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        pageBuilder.close();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Encoders.java

@@ -0,0 +1,81 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.EncoderPlugin;
+
+public abstract class Encoders
+{
+    private Encoders() { }
+
+    public static List<EncoderPlugin> newEncoderPlugins(ExecSession exec, List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<EncoderPlugin> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(exec.newPlugin(EncoderPlugin.class, config.get(PluginType.class, "type")));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources);
+    }
+
+    public static void transaction(List<EncoderPlugin> plugins, List<ConfigSource> configs,
+            Encoders.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction();
+    }
+
+    public static FileOutput open(List<EncoderPlugin> plugins, List<TaskSource> taskSources,
+            FileOutput output)
+    {
+        FileOutput out = output;
+        int pos = 0;
+        while (pos < plugins.size()) {
+            out = plugins.get(pos).open(taskSources.get(pos), out);
+            pos++;
+        }
+        return out;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<EncoderPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Encoders.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private int pos;
+
+        RecursiveControl(List<EncoderPlugin> plugins, List<ConfigSource> configs,
+                Encoders.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+        }
+
+        public void transaction()
+        {
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), new EncoderPlugin.Control() {
+                    public void run(TaskSource taskSource)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction();
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java

@@ -0,0 +1,95 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import org.embulk.config.TaskSource;
+import org.embulk.config.TaskReport;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.ProcessState;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.FilterPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.ProcessTask;
+import org.embulk.plugin.compat.PluginWrappers;
+
+public abstract class Executors
+{
+    private Executors() { }
+
+    public interface ProcessStateCallback
+    {
+        public void started();
+
+        public void inputCommitted(TaskReport report);
+
+        public void outputCommitted(TaskReport report);
+    }
+
+    public static void process(ExecSession exec,
+            ProcessTask task, int taskIndex,
+            ProcessStateCallback callback)
+    {
+        InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
+        List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
+        OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
+
+        // TODO assert task.getExecutorSchema().equals task.getOutputSchema()
+
+        process(exec, taskIndex,
+                inputPlugin, task.getInputSchema(), task.getInputTaskSource(),
+                filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources(),
+                outputPlugin, task.getOutputSchema(), task.getOutputTaskSource(),
+                callback);
+    }
+
+    public static void process(ExecSession exec, int taskIndex,
+            InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource,
+            List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources,
+            OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource,
+            ProcessStateCallback callback)
+    {
+        TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
+            outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
+
+        PageOutput closeThis = tran;
+        callback.started();
+        try {
+            PageOutput filtered = closeThis = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
+
+            TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
+
+            if (inputTaskReport == null) {
+                inputTaskReport = exec.newTaskReport();
+            }
+            callback.inputCommitted(inputTaskReport);
+
+            TaskReport outputTaskReport = tran.commit();
+            tran = null;
+            if (outputTaskReport == null) {
+                outputTaskReport = exec.newTaskReport();
+            }
+            callback.outputCommitted(outputTaskReport);  // TODO check output.finish() is called. wrap or abstract
+
+        } finally {
+            try {
+                if (tran != null) {
+                    tran.abort();
+                }
+            } finally {
+                closeThis.close();
+            }
+        }
+    }
+
+    public static Schema getInputSchema(List<Schema> schemas)
+    {
+        return schemas.get(0);
+    }
+
+    public static Schema getOutputSchema(List<Schema> schemas)
+    {
+        return schemas.get(schemas.size() - 1);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java

@@ -0,0 +1,111 @@
+package org.embulk.spi.util;
+
+import java.io.InputStream;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+
+public class FileInputInputStream
+        extends InputStream
+{
+    private final FileInput in;
+    private int pos;
+    private Buffer buffer = Buffer.EMPTY;
+
+    public FileInputInputStream(FileInput in)
+    {
+        this.in = in;
+    }
+
+    public boolean markSupported()
+    {
+        return false;
+    }
+
+    public boolean nextFile()
+    {
+        releaseBuffer();
+        return in.nextFile();
+    }
+
+    @Override
+    public int available()
+    {
+        return buffer.limit() - pos;
+    }
+
+    @Override
+    public int read()
+    {
+        while (pos >= buffer.limit()) {
+            if (!nextBuffer()) {
+                return -1;
+            }
+        }
+        byte b = buffer.array()[buffer.offset() + pos];
+        pos++;
+        if (pos >= buffer.limit()) {
+            releaseBuffer();
+        }
+        return b & 0xff;
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len)
+    {
+        while (pos >= buffer.limit()) {
+            if (!nextBuffer()) {
+                return -1;
+            }
+        }
+        int remaining = buffer.limit() - pos;
+        boolean allConsumed;
+        if (remaining <= len) {
+            allConsumed = true;
+            len = remaining;
+        } else {
+            allConsumed = false;
+        }
+        if (b != null) {
+            // b == null if skip
+            buffer.getBytes(pos, b, off, len);
+        }
+        if (allConsumed) {
+            releaseBuffer();
+        } else {
+            pos += len;
+        }
+        return len;
+    }
+
+    @Override
+    public long skip(long len)
+    {
+        int skipped = read(null, 0, (int) Math.min(len, Integer.MAX_VALUE));
+        return skipped > 0 ? skipped : 0;
+    }
+
+    private boolean nextBuffer()
+    {
+        releaseBuffer();
+        Buffer b = in.poll();
+        if (b == null) {
+            return false;
+        }
+        buffer = b;
+        return true;
+    }
+
+    private void releaseBuffer()
+    {
+        buffer.release();
+        buffer = Buffer.EMPTY;
+        pos = 0;
+    }
+
+    @Override
+    public void close()
+    {
+        releaseBuffer();
+        in.close();
+    }
+}