embulk 0.8.36-java → 0.10.25-java

data/embulk-core/src/main/java/org/embulk/exec/ResumeState.java

@@ -1,100 +0,0 @@
-package org.embulk.exec;
-
-import java.util.List;
-import com.google.common.base.Optional;
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.config.CommitReport;
-import org.embulk.spi.Schema;
-
-public class ResumeState
-{
-    private final ConfigSource execSessionConfigSource;
-    private final TaskSource inputTaskSource;
-    private final TaskSource outputTaskSource;
-    private final Schema inputSchema;
-    private final Schema outputSchema;
-    private final List<Optional<TaskReport>> inputTaskReports;
-    private final List<Optional<TaskReport>> outputTaskReports;
-
-    @JsonCreator
-    public ResumeState(
-            @JsonProperty("exec_task") ConfigSource execSessionConfigSource,
-            @JsonProperty("in_task") TaskSource inputTaskSource,
-            @JsonProperty("out_task") TaskSource outputTaskSource,
-            @JsonProperty("in_schema") Schema inputSchema,
-            @JsonProperty("out_schema") Schema outputSchema,
-            @JsonProperty("in_reports") List<Optional<TaskReport>> inputTaskReports,
-            @JsonProperty("out_reports") List<Optional<TaskReport>> outputTaskReports)
-    {
-        this.execSessionConfigSource = execSessionConfigSource;
-        this.inputTaskSource = inputTaskSource;
-        this.outputTaskSource = outputTaskSource;
-        this.inputSchema = inputSchema;
-        this.outputSchema = outputSchema;
-        this.inputTaskReports = inputTaskReports;
-        this.outputTaskReports = outputTaskReports;
-    }
-
-    @JsonProperty("exec_task")
-    public ConfigSource getExecSessionConfigSource()
-    {
-        return execSessionConfigSource;
-    }
-
-    @JsonProperty("in_task")
-    public TaskSource getInputTaskSource()
-    {
-        return inputTaskSource;
-    }
-
-    @JsonProperty("out_task")
-    public TaskSource getOutputTaskSource()
-    {
-        return outputTaskSource;
-    }
-
-    @JsonProperty("in_schema")
-    public Schema getInputSchema()
-    {
-        return inputSchema;
-    }
-
-    @JsonProperty("out_schema")
-    public Schema getOutputSchema()
-    {
-        return outputSchema;
-    }
-
-    @JsonProperty("in_reports")
-    public List<Optional<TaskReport>> getInputTaskReports()
-    {
-        return inputTaskReports;
-    }
-
-    @Deprecated
-    @JsonIgnore
-    @SuppressWarnings("unchecked")
-    public List<Optional<CommitReport>> getInputCommitReports()
-    {
-        return (List) inputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport
-    }
-
-    @JsonProperty("out_reports")
-    public List<Optional<TaskReport>> getOutputTaskReports()
-    {
-        return outputTaskReports;
-    }
-
-    @Deprecated
-    @JsonIgnore
-    @SuppressWarnings("unchecked")
-    public List<Optional<CommitReport>> getOutputCommitReports()
-    {
-        return (List) outputTaskReports;  // the only implementation of TaskReport is DataSourceImpl which implements CommitReport;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/SamplingParserPlugin.java

@@ -1,215 +0,0 @@
-package org.embulk.exec;
-
-import java.text.NumberFormat;
-import java.util.List;
-
-import com.google.inject.Inject;
-import com.google.common.base.Preconditions;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Exec;
-import org.embulk.spi.Page;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.ParserPlugin;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.FileInputRunner;
-import org.embulk.spi.PageOutput;
-import org.slf4j.Logger;
-
-import static java.util.Locale.ENGLISH;
-import static org.embulk.spi.util.Inputs.each;
-
-/*
- * Used by FileInputRunner.guess
- */
-public class SamplingParserPlugin
-        implements ParserPlugin
-{
-    public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig)
-    {
-        return runFileInputSampling(runner, inputConfig, Exec.newConfigSource());
-    }
-
-    public static Buffer runFileInputSampling(final FileInputRunner runner, ConfigSource inputConfig, ConfigSource sampleBufferConfig)
-    {
-        final SampleBufferTask sampleBufferTask = sampleBufferConfig.loadConfig(SampleBufferTask.class);
-
-        // override in.parser.type so that FileInputRunner creates SamplingParserPlugin
-        ConfigSource samplingInputConfig = inputConfig.deepCopy();
-        samplingInputConfig.getNestedOrSetEmpty("parser")
-                .set("type", "system_sampling")
-                .set("sample_buffer_bytes", sampleBufferTask.getSampleBufferBytes());
-        samplingInputConfig.set("decoders", null);
-
-        try {
-            runner.transaction(samplingInputConfig, new InputPlugin.Control() {
-                public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
-                {
-                    if (taskCount == 0) {
-                        throw new NoSampleException("No input files to read sample data");
-                    }
-                    int maxSize = -1;
-                    int maxSizeTaskIndex = -1;
-                    for (int taskIndex=0; taskIndex < taskCount; taskIndex++) {
-                        try {
-                            runner.run(taskSource, schema, taskIndex, new PageOutput() {
-                                @Override
-                                public void add(Page page)
-                                {
-                                    throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
-                                }
-
-                                public void finish() { }
-
-                                public void close() { }
-                            });
-                        } catch (NotEnoughSampleError ex) {
-                            if (maxSize < ex.getSize()) {
-                                maxSize = ex.getSize();
-                                maxSizeTaskIndex = taskIndex;
-                            }
-                            continue;
-                        }
-                    }
-                    if (maxSize <= 0) {
-                        throw new NoSampleException("All input files are empty");
-                    }
-                    taskSource.getNested("ParserTaskSource").set("force", true);
-                    try {
-                        runner.run(taskSource, schema, maxSizeTaskIndex, new PageOutput() {
-                            @Override
-                            public void add(Page page)
-                            {
-                                throw new RuntimeException("Input plugin must be a FileInputPlugin to guess parser configuration");  // TODO exception class
-                            }
-
-                            public void finish() { }
-
-                            public void close() { }
-                        });
-                    } catch (NotEnoughSampleError ex) {
-                        throw new NoSampleException("All input files are smaller than minimum sampling size");
-                    }
-                    throw new NoSampleException("All input files are smaller than minimum sampling size");
-                }
-            });
-            throw new AssertionError("SamplingParserPlugin must throw SampledNoticeError");
-        } catch (SampledNoticeError error) {
-            return error.getSample();
-        }
-    }
-
-    public static class SampledNoticeError
-            extends Error
-    {
-        private final Buffer sample;
-
-        public SampledNoticeError(Buffer sample)
-        {
-            this.sample = sample;
-        }
-
-        public Buffer getSample()
-        {
-            return sample;
-        }
-    }
-
-    public static class NotEnoughSampleError
-            extends Error
-    {
-        private final int size;
-
-        public NotEnoughSampleError(int size)
-        {
-            this.size = size;
-        }
-
-        public int getSize()
-        {
-            return size;
-        }
-    }
-
-    private final NumberFormat numberFormat = NumberFormat.getNumberInstance(ENGLISH);
-    private final Logger log = Exec.getLogger(this.getClass());
-    private final int minSampleBufferBytes;
-
-    public interface PluginTask
-            extends Task, SampleBufferTask
-    {
-    }
-
-    public interface SampleBufferTask
-            extends Task
-    {
-        @Config("sample_buffer_bytes")
-        @ConfigDefault("32768") // 32 * 1024
-        public int getSampleBufferBytes();
-    }
-
-    @Inject
-    public SamplingParserPlugin(@ForSystemConfig ConfigSource systemConfig)
-    {
-        this.minSampleBufferBytes = 40;  // empty gzip file is 33 bytes. // TODO get sample size from system config
-    }
-
-    @Override
-    public void transaction(ConfigSource config, ParserPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-        Preconditions.checkArgument(minSampleBufferBytes < task.getSampleBufferBytes(), "minSampleBufferBytes must be smaller than sample_buffer_bytes");
-
-        log.info("Try to read {} bytes from input source", numberFormat.format(task.getSampleBufferBytes()));
-        control.run(task.dump(), null);
-    }
-
-    @Override
-    public void run(TaskSource taskSource, Schema schema,
-            FileInput input, PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        Buffer buffer = readSample(input, task.getSampleBufferBytes());
-        if (!taskSource.get(boolean.class, "force", false)) {
-            if (buffer.limit() < minSampleBufferBytes) {
-                throw new NotEnoughSampleError(buffer.limit());
-            }
-        }
-        throw new SampledNoticeError(buffer);
-    }
-
-    public static Buffer readSample(FileInput fileInput, int sampleBufferBytes)
-    {
-        return readSample(fileInput, Buffer.allocate(sampleBufferBytes), 0, sampleBufferBytes);
-    }
-
-    public static Buffer readSample(FileInput fileInput, Buffer sample, int offset, int sampleBufferBytes)
-    {
-        if (!fileInput.nextFile()) {
-            // no input files
-            return sample;
-        }
-
-        try {
-            for (Buffer buffer : each(fileInput)) {
-                int size = Math.min(buffer.limit(), sample.capacity() - offset);
-                sample.setBytes(offset, buffer, 0, size);
-                offset += size;
-                buffer.release();
-                if (offset >= sampleBufferBytes) {
-                    break;
-                }
-            }
-        }
-        finally {
-            sample.limit(offset);
-        }
-        return sample;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/SetCurrentThreadName.java

@@ -1,22 +0,0 @@
-package org.embulk.exec;
-
-import static java.util.Locale.ENGLISH;
-
-public class SetCurrentThreadName
-        implements AutoCloseable
-{
-    private final String original;
-
-    public SetCurrentThreadName(String name)
-    {
-        this.original = Thread.currentThread().getName();
-        Thread thread = Thread.currentThread();
-        thread.setName(String.format(ENGLISH, "%04d:", thread.getId()) + name);
-    }
-
-    @Override
-    public void close()
-    {
-        Thread.currentThread().setName(original);
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/SkipTransactionException.java

@@ -1,23 +0,0 @@
-package org.embulk.exec;
-
-import org.embulk.config.ConfigDiff;
-
-// Input/output plugins might need to stop Embulk before the transaction starts by depending
-// on the conditions of input/output data sources/destinations. They can throw this exception
-// if they want to do that. Embulk handles it and then stops the transaction.
-public class SkipTransactionException
-        extends RuntimeException
-{
-    private final ConfigDiff configDiff;
-
-    public SkipTransactionException(ConfigDiff configDiff)
-    {
-        super();
-        this.configDiff = configDiff;
-    }
-
-    public ConfigDiff getConfigDiff()
-    {
-        return configDiff;
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/SystemConfigModule.java

@@ -1,24 +0,0 @@
-package org.embulk.exec;
-
-import com.google.inject.Module;
-import com.google.inject.Binder;
-import org.embulk.config.ConfigSource;
-
-public class SystemConfigModule
-        implements Module
-{
-    private final ConfigSource systemConfig;
-
-    public SystemConfigModule(ConfigSource systemConfig)
-    {
-        this.systemConfig = systemConfig;
-    }
-
-    @Override
-    public void configure(Binder binder)
-    {
-        binder.bind(ConfigSource.class)
-            .annotatedWith(ForSystemConfig.class)
-            .toInstance(systemConfig);
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/TempFileAllocator.java

@@ -1,35 +0,0 @@
-package org.embulk.exec;
-
-import java.io.File;
-import com.google.inject.Inject;
-import org.embulk.config.ConfigSource;
-import org.embulk.spi.TempFileSpace;
-
-// TODO change this class to interface
-// TODO don't use this class directly. Use spi.Exec.getTempFileSpace() instead.
-public class TempFileAllocator
-{
-    private final File[] dirs;
-
-    @Inject
-    public TempFileAllocator(@ForSystemConfig ConfigSource systemConfig)
-    {
-        // TODO get `temp_dirs` from system config
-        String s = System.getProperty("java.io.tmpdir");
-        if (s == null || s.isEmpty()) {
-            s = "/tmp";
-        }
-        this.dirs = new File[] {
-            new File(s, "embulk")
-        };
-    }
-
-    public TempFileSpace newSpace(String subdir)
-    {
-        // TODO support multiple directories
-        // UNIX/Linux cannot include '/' as file name.
-        // Windows cannot include ':' as file name.
-        subdir = subdir.replace('/', '-').replace(':', '-');
-        return new TempFileSpace(new File(dirs[0], subdir));
-    }
-}

data/embulk-core/src/main/java/org/embulk/exec/TransactionStage.java

@@ -1,27 +0,0 @@
-package org.embulk.exec;
-
-public enum TransactionStage
-{
-    INPUT_BEGIN(1),
-    FILTER_BEGIN(2),
-    EXECUTOR_BEGIN(3),
-    OUTPUT_BEGIN(4),
-    RUN(5),
-    OUTPUT_COMMIT(6),
-    EXECUTOR_COMMIT(7),
-    FILTER_COMMIT(8),
-    INPUT_COMMIT(9),
-    CLEANUP(10);
-
-    private final int index;
-
-    private TransactionStage(int index)
-    {
-        this.index = index;
-    }
-
-    public boolean isBefore(TransactionStage another)
-    {
-        return index < another.index;
-    }
-}