embulk 0.8.37-java → 0.10.26-java

data/embulk-core/src/main/java/org/embulk/spi/util/Executors.java

@@ -1,93 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import org.embulk.config.TaskSource;
-import org.embulk.config.TaskReport;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.ProcessState;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.FilterPlugin;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.ProcessTask;
-import org.embulk.spi.AbortTransactionResource;
-import org.embulk.spi.CloseResource;
-import org.embulk.plugin.compat.PluginWrappers;
-
-public abstract class Executors
-{
-    private Executors() { }
-
-    public interface ProcessStateCallback
-    {
-        public void started();
-
-        public void inputCommitted(TaskReport report);
-
-        public void outputCommitted(TaskReport report);
-    }
-
-    public static void process(ExecSession exec,
-            ProcessTask task, int taskIndex,
-            ProcessStateCallback callback)
-    {
-        InputPlugin inputPlugin = exec.newPlugin(InputPlugin.class, task.getInputPluginType());
-        List<FilterPlugin> filterPlugins = Filters.newFilterPlugins(exec, task.getFilterPluginTypes());
-        OutputPlugin outputPlugin = exec.newPlugin(OutputPlugin.class, task.getOutputPluginType());
-
-        // TODO assert task.getExecutorSchema().equals task.getOutputSchema()
-
-        process(exec, taskIndex,
-                inputPlugin, task.getInputSchema(), task.getInputTaskSource(),
-                filterPlugins, task.getFilterSchemas(), task.getFilterTaskSources(),
-                outputPlugin, task.getOutputSchema(), task.getOutputTaskSource(),
-                callback);
-    }
-
-    public static void process(ExecSession exec, int taskIndex,
-            InputPlugin inputPlugin, Schema inputSchema, TaskSource inputTaskSource,
-            List<FilterPlugin> filterPlugins, List<Schema> filterSchemas, List<TaskSource> filterTaskSources,
-            OutputPlugin outputPlugin, Schema outputSchema, TaskSource outputTaskSource,
-            ProcessStateCallback callback)
-    {
-        TransactionalPageOutput tran = PluginWrappers.transactionalPageOutput(
-            outputPlugin.open(outputTaskSource, outputSchema, taskIndex));
-
-        callback.started();
-        // here needs to use try-with-resource to add exception happend at close() or abort()
-        // to suppressed exception. otherwise exception happend at close() or abort() overwrites
-        // essential exception.
-        try (CloseResource closer = new CloseResource(tran)) {
-            try (AbortTransactionResource aborter = new AbortTransactionResource(tran)) {
-                PageOutput filtered = Filters.open(filterPlugins, filterTaskSources, filterSchemas, tran);
-                closer.closeThis(filtered);
-
-                TaskReport inputTaskReport = inputPlugin.run(inputTaskSource, inputSchema, taskIndex, filtered);
-
-                if (inputTaskReport == null) {
-                    inputTaskReport = exec.newTaskReport();
-                }
-                callback.inputCommitted(inputTaskReport);
-
-                TaskReport outputTaskReport = tran.commit();
-                aborter.dontAbort();
-                if (outputTaskReport == null) {
-                    outputTaskReport = exec.newTaskReport();
-                }
-                callback.outputCommitted(outputTaskReport);  // TODO check output.finish() is called. wrap or abstract
-            }
-        }
-    }
-
-    public static Schema getInputSchema(List<Schema> schemas)
-    {
-        return schemas.get(0);
-    }
-
-    public static Schema getOutputSchema(List<Schema> schemas)
-    {
-        return schemas.get(schemas.size() - 1);
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/FileInputInputStream.java

@@ -1,111 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.InputStream;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInput;
-
-public class FileInputInputStream
-        extends InputStream
-{
-    private final FileInput in;
-    private int pos;
-    private Buffer buffer = Buffer.EMPTY;
-
-    public FileInputInputStream(FileInput in)
-    {
-        this.in = in;
-    }
-
-    public boolean markSupported()
-    {
-        return false;
-    }
-
-    public boolean nextFile()
-    {
-        releaseBuffer();
-        return in.nextFile();
-    }
-
-    @Override
-    public int available()
-    {
-        return buffer.limit() - pos;
-    }
-
-    @Override
-    public int read()
-    {
-        while (pos >= buffer.limit()) {
-            if (!nextBuffer()) {
-                return -1;
-            }
-        }
-        byte b = buffer.array()[buffer.offset() + pos];
-        pos++;
-        if (pos >= buffer.limit()) {
-            releaseBuffer();
-        }
-        return b & 0xff;
-    }
-
-    @Override
-    public int read(byte[] b, int off, int len)
-    {
-        while (pos >= buffer.limit()) {
-            if (!nextBuffer()) {
-                return -1;
-            }
-        }
-        int remaining = buffer.limit() - pos;
-        boolean allConsumed;
-        if (remaining <= len) {
-            allConsumed = true;
-            len = remaining;
-        } else {
-            allConsumed = false;
-        }
-        if (b != null) {
-            // b == null if skip
-            buffer.getBytes(pos, b, off, len);
-        }
-        if (allConsumed) {
-            releaseBuffer();
-        } else {
-            pos += len;
-        }
-        return len;
-    }
-
-    @Override
-    public long skip(long len)
-    {
-        int skipped = read(null, 0, (int) Math.min(len, Integer.MAX_VALUE));
-        return skipped > 0 ? skipped : 0;
-    }
-
-    private boolean nextBuffer()
-    {
-        releaseBuffer();
-        Buffer b = in.poll();
-        if (b == null) {
-            return false;
-        }
-        buffer = b;
-        return true;
-    }
-
-    private void releaseBuffer()
-    {
-        buffer.release();
-        buffer = Buffer.EMPTY;
-        pos = 0;
-    }
-
-    @Override
-    public void close()
-    {
-        releaseBuffer();
-        in.close();
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java

@@ -1,119 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.OutputStream;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.FileOutput;
-
-public class FileOutputOutputStream
-        extends OutputStream
-{
-    private final FileOutput out;
-    private final BufferAllocator allocator;
-    private final CloseMode closeMode;
-    private int pos;
-    private Buffer buffer;
-
-    public static enum CloseMode {
-        FLUSH,
-        FLUSH_FINISH,
-        FLUSH_FINISH_CLOSE,
-        CLOSE;
-    }
-
-    public FileOutputOutputStream(FileOutput out, BufferAllocator allocator, CloseMode closeMode)
-    {
-        this.out = out;
-        this.allocator = allocator;
-        this.buffer = allocator.allocate();
-        this.closeMode = closeMode;
-    }
-
-    public void nextFile()
-    {
-        out.nextFile();
-    }
-
-    public void finish()
-    {
-        doFlush();
-        out.finish();
-    }
-
-    @Override
-    public void write(int b)
-    {
-        buffer.array()[buffer.offset() + pos] = (byte) b;
-        pos++;
-        if (pos >= buffer.capacity()) {
-            flush();
-        }
-    }
-
-    @Override
-    public void write(byte[] b, int off, int len)
-    {
-        while (true) {
-            int available = buffer.capacity() - pos;
-            if (available < len) {
-                buffer.setBytes(pos, b, off, available);
-                pos += available;
-                len -= available;
-                off += available;
-                flush();
-            } else {
-                buffer.setBytes(pos, b, off, len);
-                pos += len;
-                if (available <= len) {
-                    flush();
-                }
-                break;
-            }
-        }
-    }
-
-    private boolean doFlush()
-    {
-        if (pos > 0) {
-            buffer.limit(pos);
-            out.add(buffer);
-            buffer = Buffer.EMPTY;
-            pos = 0;
-            return true;
-        }
-        return false;
-    }
-
-    @Override
-    public void flush()
-    {
-        if (doFlush()) {
-            buffer = allocator.allocate();
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        switch (closeMode) {
-        case FLUSH:
-            doFlush();
-            break;
-        case FLUSH_FINISH:
-            doFlush();
-            out.finish();
-            break;
-        case FLUSH_FINISH_CLOSE:
-            doFlush();
-            out.finish();
-            out.close();
-            break;
-        case CLOSE:
-            out.close();
-            break;
-        }
-        buffer.release();
-        buffer = Buffer.EMPTY;
-        pos = 0;
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -1,100 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.plugin.PluginType;
-import org.embulk.spi.ExecSession;
-import org.embulk.spi.Schema;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.FilterPlugin;
-
-public abstract class Filters
-{
-    private Filters() { }
-
-    public static List<PluginType> getPluginTypes(List<ConfigSource> configs)
-    {
-        ImmutableList.Builder<PluginType> builder = ImmutableList.builder();
-        for (ConfigSource config : configs) {
-            builder.add(config.get(PluginType.class, "type"));
-        }
-        return builder.build();
-    }
-
-    public static List<FilterPlugin> newFilterPluginsFromConfigSources(ExecSession exec, List<ConfigSource> configs)
-    {
-        return newFilterPlugins(exec, getPluginTypes(configs));
-    }
-
-    public static List<FilterPlugin> newFilterPlugins(ExecSession exec, List<PluginType> pluginTypes)
-    {
-        ImmutableList.Builder<FilterPlugin> builder = ImmutableList.builder();
-        for (PluginType pluginType : pluginTypes) {
-            builder.add(exec.newPlugin(FilterPlugin.class, pluginType));
-        }
-        return builder.build();
-    }
-
-    public interface Control
-    {
-        public void run(List<TaskSource> taskSources, List<Schema> filterSchemas);
-    }
-
-    public static void transaction(List<FilterPlugin> plugins, List<ConfigSource> configs,
-            Schema inputSchema, Filters.Control control)
-    {
-        new RecursiveControl(plugins, configs, control).transaction(inputSchema);
-    }
-
-    public static PageOutput open(List<FilterPlugin> plugins, List<TaskSource> taskSources,
-            List<Schema> filterSchemas, PageOutput output)
-    {
-        PageOutput out = output;
-        int pos = plugins.size() - 1;
-        while (pos >= 0) {
-            out = plugins.get(pos).open(taskSources.get(pos), filterSchemas.get(pos), filterSchemas.get(pos + 1), out);
-            pos--;
-        }
-        return out;
-    }
-
-    private static class RecursiveControl
-    {
-        private final List<FilterPlugin> plugins;
-        private final List<ConfigSource> configs;
-        private final Filters.Control finalControl;
-        private final ImmutableList.Builder<TaskSource> taskSources;
-        private final ImmutableList.Builder<Schema> filterSchemas;
-        private int pos;
-
-        RecursiveControl(List<FilterPlugin> plugins, List<ConfigSource> configs,
-                Filters.Control finalControl)
-        {
-            this.plugins = plugins;
-            this.configs = configs;
-            this.finalControl = finalControl;
-            this.taskSources = ImmutableList.builder();
-            this.filterSchemas = ImmutableList.builder();
-        }
-
-        public void transaction(Schema inputSchema)
-        {
-            filterSchemas.add(inputSchema);
-            if (pos < plugins.size()) {
-                plugins.get(pos).transaction(configs.get(pos), inputSchema, new FilterPlugin.Control() {
-                    public void run(TaskSource taskSource, Schema outputSchema)
-                    {
-                        taskSources.add(taskSource);
-                        pos++;
-                        transaction(outputSchema);
-                    }
-                });
-            } else {
-                finalControl.run(taskSources.build(), filterSchemas.build());
-            }
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java

@@ -1,189 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.io.InputStream;
-import java.io.Closeable;
-import java.io.IOException;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.BufferAllocator;
-
-public class InputStreamFileInput
-        implements FileInput
-{
-    public interface Provider extends Closeable
-    {
-        public InputStream openNext() throws IOException;
-
-        public void close() throws IOException;
-    }
-
-    public interface Opener
-    {
-        public InputStream open() throws IOException;
-    }
-
-    public static class IteratorProvider implements Provider
-    {
-        private Iterator<InputStream> iterator;
-
-        public IteratorProvider(Iterable<InputStream> iterable)
-        {
-            this.iterator = iterable.iterator();
-        }
-
-        public IteratorProvider(Iterator<InputStream> iterator)
-        {
-            this.iterator = iterator;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (!iterator.hasNext()) {
-                return null;
-            }
-            return iterator.next();
-        }
-
-        @Override
-        public void close() throws IOException
-        {
-            while (iterator.hasNext()) {
-                iterator.next().close();
-            }
-        }
-    }
-
-    private static class OpenerProvider implements Provider
-    {
-        private Opener opener;
-
-        public OpenerProvider(Opener opener)
-        {
-            this.opener = opener;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (opener == null) {
-                return null;
-            }
-            InputStream stream = opener.open();
-            opener = null;
-            return stream;
-        }
-
-        @Override
-        public void close() throws IOException
-        { }
-    }
-
-    private static class InputStreamProvider implements Provider
-    {
-        private InputStream input;
-
-        public InputStreamProvider(InputStream input)
-        {
-            this.input = input;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (input == null) {
-                return null;
-            }
-            InputStream ret = input;
-            input = null;
-            return ret;
-        }
-
-        @Override
-        public void close() throws IOException
-        {
-            if (input != null) {
-                input.close();
-                input = null;
-            }
-        }
-    }
-
-    private final BufferAllocator allocator;
-    private final Provider provider;
-    private InputStream current;
-
-    public InputStreamFileInput(BufferAllocator allocator, Provider provider)
-    {
-        this.allocator = allocator;
-        this.provider = provider;
-        this.current = null;
-    }
-
-    public InputStreamFileInput(BufferAllocator allocator, Opener opener)
-    {
-        this(allocator, new OpenerProvider(opener));
-    }
-
-    public InputStreamFileInput(BufferAllocator allocator, InputStream openedStream)
-    {
-        this(allocator, new InputStreamProvider(openedStream));
-    }
-
-    public Buffer poll()
-    {
-        if (current == null) {
-            throw new IllegalStateException("nextFile() must be called before poll()");
-        }
-        Buffer buffer = allocator.allocate();
-        try {
-            int n = current.read(buffer.array(), buffer.offset(), buffer.capacity());
-            if (n < 0) {
-                return null;
-            }
-            buffer.limit(n);
-            Buffer b = buffer;
-            buffer = null;
-            return b;
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        } finally {
-            if (buffer != null) {
-                buffer.release();
-                buffer = null;
-            }
-        }
-    }
-
-    public boolean nextFile()
-    {
-        try {
-            if (current != null) {
-                current.close();
-                current = null;
-            }
-            current = provider.openNext();
-            return current != null;
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void close()
-    {
-        try {
-            try {
-                if (current != null) {
-                    current.close();
-                    current = null;
-                }
-            } finally {
-                provider.close();
-            }
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-}