embulk 0.8.39-java → 0.10.28-java

data/embulk-core/src/main/java/org/embulk/spi/FileOutput.java

@@ -1,13 +0,0 @@
-package org.embulk.spi;
-
-public interface FileOutput
-        extends AutoCloseable
-{
-    void nextFile();
-
-    void add(Buffer buffer);
-
-    void finish();
-
-    void close();
-}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputPlugin.java

@@ -1,28 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-
-public interface FileOutputPlugin
-{
-    interface Control
-    {
-        List<TaskReport> run(TaskSource taskSource);
-    }
-
-    ConfigDiff transaction(ConfigSource config, int taskCount,
-            FileOutputPlugin.Control control);
-
-    ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileOutputPlugin.Control control);
-
-    void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports);
-
-    TransactionalFileOutput open(TaskSource taskSource, int taskIndex);
-}

data/embulk-core/src/main/java/org/embulk/spi/FileOutputRunner.java

@@ -1,199 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import java.util.ArrayList;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.plugin.PluginType;
-import org.embulk.plugin.compat.PluginWrappers;
-import org.embulk.spi.util.Encoders;
-
-public class FileOutputRunner
-        implements OutputPlugin
-{
-    private final FileOutputPlugin fileOutputPlugin;
-
-    public FileOutputRunner(FileOutputPlugin fileOutputPlugin)
-    {
-        this.fileOutputPlugin = fileOutputPlugin;
-    }
-
-    private interface RunnerTask extends Task
-    {
-        @Config("type")
-        public PluginType getType();
-
-        @Config("encoders")
-        @ConfigDefault("[]")
-        public List<ConfigSource> getEncoderConfigs();
-
-        @Config("formatter")
-        public ConfigSource getFormatterConfig();
-
-        public void setFileOutputTaskSource(TaskSource v);
-        public TaskSource getFileOutputTaskSource();
-
-        public void setEncoderTaskSources(List<TaskSource> v);
-        public List<TaskSource> getEncoderTaskSources();
-
-        public void setFormatterTaskSource(TaskSource v);
-        public TaskSource getFormatterTaskSource();
-    }
-
-    protected List<EncoderPlugin> newEncoderPlugins(RunnerTask task)
-    {
-        return Encoders.newEncoderPlugins(Exec.session(), task.getEncoderConfigs());
-    }
-
-    protected FormatterPlugin newFormatterPlugin(RunnerTask task)
-    {
-        return Exec.newPlugin(FormatterPlugin.class, task.getFormatterConfig().get(PluginType.class, "type"));
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            final Schema schema, final int taskCount,
-            final OutputPlugin.Control control)
-    {
-        final RunnerTask task = config.loadConfig(RunnerTask.class);
-        return fileOutputPlugin.transaction(config, taskCount, new RunnerControl(schema, task, control));
-    }
-
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            final OutputPlugin.Control control)
-    {
-        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        return fileOutputPlugin.resume(task.getFileOutputTaskSource(), taskCount, new RunnerControl(schema, task, control));
-    }
-
-    private class RunnerControl
-            implements FileOutputPlugin.Control
-    {
-        private final Schema schema;
-        private final RunnerTask task;
-        private final List<EncoderPlugin> encoderPlugins;
-        private final FormatterPlugin formatterPlugin;
-        private final OutputPlugin.Control nextControl;
-
-        public RunnerControl(Schema schema, RunnerTask task, OutputPlugin.Control nextControl)
-        {
-            this.schema = schema;
-            this.task = task;
-            // create plugins earlier than run() to throw exceptions early
-            this.encoderPlugins = newEncoderPlugins(task);
-            this.formatterPlugin = newFormatterPlugin(task);
-            this.nextControl = nextControl;
-        }
-
-        @Override
-        public List<TaskReport> run(final TaskSource fileOutputTaskSource)
-        {
-            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
-            Encoders.transaction(encoderPlugins, task.getEncoderConfigs(), new Encoders.Control() {
-                public void run(final List<TaskSource> encoderTaskSources)
-                {
-                    formatterPlugin.transaction(task.getFormatterConfig(), schema, new FormatterPlugin.Control() {
-                        public void run(final TaskSource formatterTaskSource)
-                        {
-                            task.setFileOutputTaskSource(fileOutputTaskSource);
-                            task.setEncoderTaskSources(encoderTaskSources);
-                            task.setFormatterTaskSource(formatterTaskSource);
-                            taskReports.addAll(nextControl.run(task.dump()));
-                        }
-                    });
-                }
-            });
-            return taskReports;
-        }
-    }
-
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successtaskReports)
-    {
-        fileOutputPlugin.cleanup(taskSource, taskCount, successtaskReports);
-    }
-
-    @Override
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex)
-    {
-        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        List<EncoderPlugin> encoderPlugins = newEncoderPlugins(task);
-        FormatterPlugin formatterPlugin = newFormatterPlugin(task);
-
-        try (AbortTransactionResource aborter = new AbortTransactionResource()) {
-            try (CloseResource closer = new CloseResource()) {
-                TransactionalFileOutput finalOutput = PluginWrappers.transactionalFileOutput(
-                        fileOutputPlugin.open(task.getFileOutputTaskSource(), taskIndex));
-                aborter.abortThis(finalOutput);
-                closer.closeThis(finalOutput);
-
-                FileOutput encodedOutput = Encoders.open(encoderPlugins, task.getEncoderTaskSources(), finalOutput);
-                closer.closeThis(encodedOutput);
-
-                PageOutput output = formatterPlugin.open(task.getFormatterTaskSource(), schema, encodedOutput);
-                closer.closeThis(output);
-
-                TransactionalPageOutput ret = new DelegateTransactionalPageOutput(finalOutput, output);
-                aborter.dontAbort();
-                closer.dontClose();  // ownership of output is transferred to caller (input plugin). the owner will close output.
-                return ret;
-            }
-        }
-    }
-
-    private static class DelegateTransactionalPageOutput
-            implements TransactionalPageOutput
-    {
-        private final Transactional tran;
-        private final PageOutput output;
-
-        public DelegateTransactionalPageOutput(Transactional tran, PageOutput output)
-        {
-            this.tran = tran;
-            this.output = output;
-        }
-
-        @Override
-        public void add(Page page)
-        {
-            output.add(page);
-        }
-
-        @Override
-        public void finish()
-        {
-            output.finish();
-        }
-
-        @Override
-        public void close()
-        {
-            output.close();
-        }
-
-        @Override
-        public void abort()
-        {
-            tran.abort();
-        }
-
-        @Override
-        public TaskReport commit()
-        {
-            // TODO check finished
-            return tran.commit();
-        }
-    }
-
-    public static TaskSource getFileOutputTaskSource(TaskSource runnerTaskSource)
-    {
-        return runnerTaskSource.loadTask(RunnerTask.class).getFileOutputTaskSource();
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/FilterPlugin.java

@@ -1,18 +0,0 @@
-package org.embulk.spi;
-
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-
-public interface FilterPlugin
-{
-    interface Control
-    {
-        void run(TaskSource taskSource, Schema outputSchema);
-    }
-
-    void transaction(ConfigSource config, Schema inputSchema,
-            FilterPlugin.Control control);
-
-    PageOutput open(TaskSource taskSource, Schema inputSchema,
-            Schema outputSchema, PageOutput output);
-}

data/embulk-core/src/main/java/org/embulk/spi/FormatterPlugin.java

@@ -1,18 +0,0 @@
-package org.embulk.spi;
-
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-
-public interface FormatterPlugin
-{
-    interface Control
-    {
-        void run(TaskSource taskSource);
-    }
-
-    void transaction(ConfigSource config, Schema schema,
-            FormatterPlugin.Control control);
-
-    PageOutput open(TaskSource taskSource, Schema schema,
-            FileOutput output);
-}

data/embulk-core/src/main/java/org/embulk/spi/GuessPlugin.java

@@ -1,9 +0,0 @@
-package org.embulk.spi;
-
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-
-public interface GuessPlugin
-{
-    ConfigDiff guess(ConfigSource config, Buffer sample);
-}

data/embulk-core/src/main/java/org/embulk/spi/InputPlugin.java

@@ -1,33 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-
-public interface InputPlugin
-{
-    interface Control
-    {
-        List<TaskReport> run(TaskSource taskSource,
-                Schema schema, int taskCount);
-    }
-
-    ConfigDiff transaction(ConfigSource config,
-            InputPlugin.Control control);
-
-    ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control);
-
-    void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports);
-
-    TaskReport run(TaskSource taskSource,
-            Schema schema, int taskIndex,
-            PageOutput output);
-
-    ConfigDiff guess(ConfigSource config);
-}

data/embulk-core/src/main/java/org/embulk/spi/OutputPlugin.java

@@ -1,29 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-
-public interface OutputPlugin
-{
-    interface Control
-    {
-        List<TaskReport> run(TaskSource taskSource);
-    }
-
-    ConfigDiff transaction(ConfigSource config,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control);
-
-    ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control);
-
-    void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports);
-
-    TransactionalPageOutput open(TaskSource taskSource, Schema schema, int taskIndex);
-}

data/embulk-core/src/main/java/org/embulk/spi/Page.java

@@ -1,86 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import org.msgpack.value.ImmutableValue;
-
-/**
- * Page is an in-process (in-JVM) container of data records.
- *
- * It serializes records to byte[] (in org.embulk.spi.Buffer) in order to:
- * A) Avoid slowness by handling many Java Objects
- * B) Avoid complexity by type-safe primitive arrays
- * C) Track memory consumption by records
- * D) Use off-heap memory
- *
- * (C) and (D) may not be so meaningful as of v0.7+ (or since earlier) as recent Embulk unlikely
- * allocates so many Pages at the same time. Recent Embulk is streaming-driven instead of
- * multithreaded queue-based.
- *
- * Page is NOT for inter-process communication. For multi-process execution such as MapReduce
- * Executor, the executor plugin takes responsibility about interoperable serialization.
- */
-public class Page
-{
-    private final Buffer buffer;
-    private List<String> stringReferences;
-    private List<ImmutableValue> valueReferences;
-
-    protected Page(Buffer buffer)
-    {
-        this.buffer = buffer;
-    }
-
-    public static Page allocate(int length)
-    {
-        return new Page(Buffer.allocate(length));
-    }
-
-    public static Page wrap(Buffer buffer)
-    {
-        return new Page(buffer);
-    }
-
-    public Page setStringReferences(List<String> values)
-    {
-        this.stringReferences = values;
-        return this;
-    }
-
-    public Page setValueReferences(List<ImmutableValue> values)
-    {
-        this.valueReferences = values;
-        return this;
-    }
-
-    public List<String> getStringReferences()
-    {
-        // TODO used by mapreduce executor
-        return stringReferences;
-    }
-
-    public List<ImmutableValue> getValueReferences()
-    {
-        // TODO used by mapreduce executor
-        return valueReferences;
-    }
-
-    public String getStringReference(int index)
-    {
-        return stringReferences.get(index);
-    }
-
-    public ImmutableValue getValueReference(int index)
-    {
-        return valueReferences.get(index);
-    }
-
-    public void release()
-    {
-        buffer.release();
-    }
-
-    public Buffer buffer()
-    {
-        return buffer;
-    }
-}