embulk 0.8.37-java → 0.10.26-java

data/embulk-core/src/main/java/org/embulk/spi/ExecAction.java

@@ -1,6 +0,0 @@
-package org.embulk.spi;
-
-public interface ExecAction <T>
-{
-    T run() throws Exception;
-}

data/embulk-core/src/main/java/org/embulk/spi/ExecSession.java

@@ -1,220 +0,0 @@
-package org.embulk.spi;
-
-import org.joda.time.DateTimeZone;
-import org.slf4j.Logger;
-import org.slf4j.ILoggerFactory;
-import com.google.common.base.Optional;
-import com.google.inject.Injector;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ModelManager;
-import org.embulk.config.CommitReport;
-import org.embulk.config.TaskReport;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskSource;
-import org.embulk.config.DataSourceImpl;
-import org.embulk.exec.TempFileAllocator;
-import org.embulk.plugin.PluginType;
-import org.embulk.plugin.PluginManager;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-
-public class ExecSession
-{
-    private final Injector injector;
-    private final ILoggerFactory loggerFactory;
-    private final ModelManager modelManager;
-    private final PluginManager pluginManager;
-    private final BufferAllocator bufferAllocator;
-
-    private final Timestamp transactionTime;
-    private final TempFileSpace tempFileSpace;
-
-    private final boolean preview;
-
-    @Deprecated
-    public interface SessionTask
-            extends Task
-    {
-        @Config("transaction_time")
-        @ConfigDefault("null")
-        Optional<Timestamp> getTransactionTime();
-    }
-
-    public static class Builder
-    {
-        private final Injector injector;
-        private ILoggerFactory loggerFactory;
-        private Timestamp transactionTime;
-
-        public Builder(Injector injector)
-        {
-            this.injector = injector;
-        }
-
-        public Builder fromExecConfig(ConfigSource configSource)
-        {
-            this.transactionTime = configSource.get(Timestamp.class, "transaction_time", null);
-            return this;
-        }
-
-        public Builder setLoggerFactory(ILoggerFactory loggerFactory)
-        {
-            this.loggerFactory = loggerFactory;
-            return this;
-        }
-
-        public Builder setTransactionTime(Timestamp timestamp)
-        {
-            this.transactionTime = timestamp;
-            return this;
-        }
-
-        public ExecSession build()
-        {
-            if (transactionTime == null) {
-                transactionTime = Timestamp.ofEpochMilli(System.currentTimeMillis());  // TODO get nanoseconds for default
-            }
-            return new ExecSession(injector, transactionTime, Optional.fromNullable(loggerFactory));
-        }
-    }
-
-    public static Builder builder(Injector injector)
-    {
-        return new Builder(injector);
-    }
-
-    @Deprecated
-    public ExecSession(Injector injector, ConfigSource configSource)
-    {
-        this(injector,
-                configSource.loadConfig(SessionTask.class).getTransactionTime().or(
-                    Timestamp.ofEpochMilli(System.currentTimeMillis())
-                    ), // TODO get nanoseconds for default
-                null);
-    }
-
-    private ExecSession(Injector injector, Timestamp transactionTime, Optional<ILoggerFactory> loggerFactory)
-    {
-        this.injector = injector;
-        this.loggerFactory = loggerFactory.or(injector.getInstance(ILoggerFactory.class));
-        this.modelManager = injector.getInstance(ModelManager.class);
-        this.pluginManager = injector.getInstance(PluginManager.class);
-        this.bufferAllocator = injector.getInstance(BufferAllocator.class);
-
-        this.transactionTime = transactionTime;
-
-        TempFileAllocator tempFileAllocator = injector.getInstance(TempFileAllocator.class);
-        this.tempFileSpace = tempFileAllocator.newSpace(transactionTime.toString());
-
-        this.preview = false;
-    }
-
-    private ExecSession(ExecSession copy, boolean preview)
-    {
-        this.injector = copy.injector;
-        this.loggerFactory = copy.loggerFactory;
-        this.modelManager = copy.modelManager;
-        this.pluginManager = copy.pluginManager;
-        this.bufferAllocator = copy.bufferAllocator;
-
-        this.transactionTime = copy.transactionTime;
-        this.tempFileSpace = copy.tempFileSpace;
-
-        this.preview = preview;
-    }
-
-    public ExecSession forPreview()
-    {
-        return new ExecSession(this, true);
-    }
-
-    public ConfigSource getSessionExecConfig()
-    {
-        return newConfigSource()
-            .set("transaction_time", transactionTime);
-    }
-
-    public Injector getInjector()
-    {
-        return injector;
-    }
-
-    public Timestamp getTransactionTime()
-    {
-        return transactionTime;
-    }
-
-    public Logger getLogger(String name)
-    {
-        return loggerFactory.getLogger(name);
-    }
-
-    public Logger getLogger(Class<?> name)
-    {
-        return loggerFactory.getLogger(name.getName());
-    }
-
-    public BufferAllocator getBufferAllocator()
-    {
-        return bufferAllocator;
-    }
-
-    public ModelManager getModelManager()
-    {
-        return modelManager;
-    }
-
-    public <T> T newPlugin(Class<T> iface, PluginType type)
-    {
-        return pluginManager.newPlugin(iface, type);
-    }
-
-    public TaskReport newTaskReport()
-    {
-        return new DataSourceImpl(modelManager);
-    }
-
-    @Deprecated
-    public CommitReport newCommitReport()
-    {
-        return new DataSourceImpl(modelManager);
-    }
-
-    public ConfigDiff newConfigDiff()
-    {
-        return new DataSourceImpl(modelManager);
-    }
-
-    public ConfigSource newConfigSource()
-    {
-        return new DataSourceImpl(modelManager);
-    }
-
-    public TaskSource newTaskSource()
-    {
-        return new DataSourceImpl(modelManager);
-    }
-
-    public TimestampFormatter newTimestampFormatter(String format, DateTimeZone timezone)
-    {
-        return new TimestampFormatter(format, timezone);
-    }
-
-    public TempFileSpace getTempFileSpace()
-    {
-        return tempFileSpace;
-    }
-
-    public boolean isPreview()
-    {
-        return preview;
-    }
-
-    public void cleanup()
-    {
-        tempFileSpace.cleanup();
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/ExecutorPlugin.java

@@ -1,19 +0,0 @@
-package org.embulk.spi;
-
-import org.embulk.config.ConfigSource;
-
-public interface ExecutorPlugin
-{
-    interface Executor
-    {
-        void execute(ProcessTask task, ProcessState state);
-    }
-
-    interface Control
-    {
-        void transaction(Schema executorSchema, int outputTaskCount, Executor executor);
-    }
-
-    void transaction(ConfigSource config, Schema outputSchema, int inputTaskCount,
-            ExecutorPlugin.Control control);
-}

data/embulk-core/src/main/java/org/embulk/spi/Extension.java

@@ -1,44 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import com.google.inject.Module;
-import org.embulk.config.ConfigSource;
-
-/**
- * Extension is a module to extend the execution framework using Guice.
- * Unlike plugins, extensions can overwrite or add core components such as
- * BufferManager, PluginSource, etc.
- * Extension is not designed for users but for framework developers to make
- * core components loosely coupled.
- *
- * An example extension to add a custom PluginSource will be as following:
- *
- * <code>
- * class MyPluginSourceExtension
- *         implements Extension, Module
- * {
- *     public static class MyPluginSource
- *             implements PluginSource
- *     {
- *         // ...
- *     }
- *
- *     {@literal @}Override
- *     public void configure(Binder binder)
- *     {
- *         Multibinder&lt;PluginSource&gt; multibinder = Multibinder.newSetBinder(binder, PluginSource.class);
- *         multibinder.addBinding().to(MyPluginSource.class);
- *     }
- *
- *     {@literal @}Override
- *     public List&lt;Module&gt; getModules()
- *     {
- *         return ImmutableList.&lt;Module&gt;of(this);
- *     }
- * }
- * </code>
- */
-public interface Extension
-{
-    List<Module> getModules(ConfigSource systemConfig);
-}

data/embulk-core/src/main/java/org/embulk/spi/FileInput.java

@@ -1,11 +0,0 @@
-package org.embulk.spi;
-
-public interface FileInput
-        extends AutoCloseable
-{
-    boolean nextFile();
-
-    Buffer poll();
-
-    void close();
-}

data/embulk-core/src/main/java/org/embulk/spi/FileInputPlugin.java

@@ -1,30 +0,0 @@
-package org.embulk.spi;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-
-public interface FileInputPlugin
-{
-    interface Control
-    {
-        List<TaskReport> run(TaskSource taskSource,
-                int taskCount);
-    }
-
-    ConfigDiff transaction(ConfigSource config,
-            FileInputPlugin.Control control);
-
-    ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control);
-
-    void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports);
-
-    TransactionalFileInput open(TaskSource taskSource,
-            int taskIndex);
-}

data/embulk-core/src/main/java/org/embulk/spi/FileInputRunner.java

@@ -1,169 +0,0 @@
-package org.embulk.spi;
-
-import java.util.ArrayList;
-import java.util.List;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.plugin.PluginType;
-import org.embulk.plugin.compat.PluginWrappers;
-import org.embulk.spi.util.Decoders;
-import org.embulk.exec.GuessExecutor;
-import org.embulk.exec.SamplingParserPlugin;
-import org.embulk.exec.ConfigurableGuessInputPlugin;
-import org.embulk.exec.NoSampleException;
-
-import static org.embulk.exec.GuessExecutor.createSampleBufferConfigFromExecConfig;
-
-public class FileInputRunner
-        implements InputPlugin, ConfigurableGuessInputPlugin
-{
-    private final FileInputPlugin fileInputPlugin;
-
-    public FileInputRunner(FileInputPlugin fileInputPlugin)
-    {
-        this.fileInputPlugin = fileInputPlugin;
-    }
-
-    private interface RunnerTask extends Task
-    {
-        // TODO "type" needed?
-
-        @Config("decoders")
-        @ConfigDefault("[]")
-        public List<ConfigSource> getDecoderConfigs();
-
-        @Config("parser")
-        public ConfigSource getParserConfig();
-
-        public void setFileInputTaskSource(TaskSource v);
-        public TaskSource getFileInputTaskSource();
-
-        public void setDecoderTaskSources(List<TaskSource> v);
-        public List<TaskSource> getDecoderTaskSources();
-
-        public void setParserTaskSource(TaskSource v);
-        public TaskSource getParserTaskSource();
-    }
-
-    protected List<DecoderPlugin> newDecoderPlugins(RunnerTask task)
-    {
-        return Decoders.newDecoderPlugins(Exec.session(), task.getDecoderConfigs());
-    }
-
-    protected ParserPlugin newParserPlugin(RunnerTask task)
-    {
-        return Exec.newPlugin(ParserPlugin.class, task.getParserConfig().get(PluginType.class, "type"));
-    }
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config, final InputPlugin.Control control)
-    {
-        final RunnerTask task = config.loadConfig(RunnerTask.class);
-        return fileInputPlugin.transaction(config, new RunnerControl(task, control));
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control)
-    {
-        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        return fileInputPlugin.resume(task.getFileInputTaskSource(), taskCount, new RunnerControl(task, control));
-    }
-
-    @Override
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return guess(Exec.newConfigSource(), config);
-    }
-
-    public ConfigDiff guess(ConfigSource execConfig, ConfigSource inputConfig)
-    {
-        final ConfigSource sampleBufferConfig = createSampleBufferConfigFromExecConfig(execConfig);
-        final Buffer sample = SamplingParserPlugin.runFileInputSampling(this, inputConfig, sampleBufferConfig);
-        // SamplingParserPlugin.runFileInputSampling throws NoSampleException if there're
-        // no files or all files are smaller than minSampleSize (40 bytes).
-
-        GuessExecutor guessExecutor = Exec.getInjector().getInstance(GuessExecutor.class);
-        return guessExecutor.guessParserConfig(sample, inputConfig, execConfig);
-    }
-
-    private class RunnerControl
-            implements FileInputPlugin.Control
-    {
-        private final RunnerTask task;
-        private final List<DecoderPlugin> decoderPlugins;
-        private final ParserPlugin parserPlugin;
-        private final InputPlugin.Control nextControl;
-
-        public RunnerControl(RunnerTask task, InputPlugin.Control nextControl)
-        {
-            this.task = task;
-            // create plugins earlier than run() to throw exceptions early
-            this.decoderPlugins = newDecoderPlugins(task);
-            this.parserPlugin = newParserPlugin(task);
-            this.nextControl = nextControl;
-        }
-
-        @Override
-        public List<TaskReport> run(final TaskSource fileInputTaskSource, final int taskCount)
-        {
-            final List<TaskReport> taskReports = new ArrayList<TaskReport>();
-            Decoders.transaction(decoderPlugins, task.getDecoderConfigs(), new Decoders.Control() {
-                public void run(final List<TaskSource> decoderTaskSources)
-                {
-                    parserPlugin.transaction(task.getParserConfig(), new ParserPlugin.Control() {
-                        public void run(final TaskSource parserTaskSource, final Schema schema)
-                        {
-                            task.setFileInputTaskSource(fileInputTaskSource);
-                            task.setDecoderTaskSources(decoderTaskSources);
-                            task.setParserTaskSource(parserTaskSource);
-                            taskReports.addAll(nextControl.run(task.dump(), schema, taskCount));
-                        }
-                    });
-                }
-            });
-            return taskReports;
-        }
-    }
-
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    {
-        fileInputPlugin.cleanup(taskSource, taskCount, successTaskReports);
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource, Schema schema, int taskIndex,
-            PageOutput output)
-    {
-        final RunnerTask task = taskSource.loadTask(RunnerTask.class);
-        List<DecoderPlugin> decoderPlugins = newDecoderPlugins(task);
-        ParserPlugin parserPlugin = newParserPlugin(task);
-
-        TransactionalFileInput tran = PluginWrappers.transactionalFileInput(
-                fileInputPlugin.open(task.getFileInputTaskSource(), taskIndex));
-        try (CloseResource closer = new CloseResource(tran)) {
-            try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
-                FileInput fileInput = Decoders.open(decoderPlugins, task.getDecoderTaskSources(), tran);
-                closer.closeThis(fileInput);
-                parserPlugin.run(task.getParserTaskSource(), schema, fileInput, output);
-
-                TaskReport report = tran.commit();  // TODO check output.finish() is called. wrap
-                aborter.dontAbort();
-                return report;
-            }
-        }
-    }
-
-    public static TaskSource getFileInputTaskSource(TaskSource runnerTaskSource)
-    {
-        return runnerTaskSource.loadTask(RunnerTask.class).getFileInputTaskSource();
-    }
-}