embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/util/FileOutputOutputStream.java

@@ -0,0 +1,119 @@
+package org.embulk.spi.util;
+
+import java.io.OutputStream;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.FileOutput;
+
+public class FileOutputOutputStream
+        extends OutputStream
+{
+    private final FileOutput out;
+    private final BufferAllocator allocator;
+    private final CloseMode closeMode;
+    private int pos;
+    private Buffer buffer;
+
+    public static enum CloseMode {
+        FLUSH,
+        FLUSH_FINISH,
+        FLUSH_FINISH_CLOSE,
+        CLOSE;
+    }
+
+    public FileOutputOutputStream(FileOutput out, BufferAllocator allocator, CloseMode closeMode)
+    {
+        this.out = out;
+        this.allocator = allocator;
+        this.buffer = allocator.allocate();
+        this.closeMode = closeMode;
+    }
+
+    public void nextFile()
+    {
+        out.nextFile();
+    }
+
+    public void finish()
+    {
+        doFlush();
+        out.finish();
+    }
+
+    @Override
+    public void write(int b)
+    {
+        buffer.array()[buffer.offset() + pos] = (byte) b;
+        pos++;
+        if (pos >= buffer.capacity()) {
+            flush();
+        }
+    }
+
+    @Override
+    public void write(byte[] b, int off, int len)
+    {
+        while (true) {
+            int available = buffer.capacity() - pos;
+            if (available < len) {
+                buffer.setBytes(pos, b, off, available);
+                pos += available;
+                len -= available;
+                off += available;
+                flush();
+            } else {
+                buffer.setBytes(pos, b, off, len);
+                pos += len;
+                if (available <= len) {
+                    flush();
+                }
+                break;
+            }
+        }
+    }
+
+    private boolean doFlush()
+    {
+        if (pos > 0) {
+            buffer.limit(pos);
+            out.add(buffer);
+            buffer = Buffer.EMPTY;
+            pos = 0;
+            return true;
+        }
+        return false;
+    }
+
+    @Override
+    public void flush()
+    {
+        if (doFlush()) {
+            buffer = allocator.allocate();
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        switch (closeMode) {
+        case FLUSH:
+            doFlush();
+            break;
+        case FLUSH_FINISH:
+            doFlush();
+            out.finish();
+            break;
+        case FLUSH_FINISH_CLOSE:
+            doFlush();
+            out.finish();
+            out.close();
+            break;
+        case CLOSE:
+            out.close();
+            break;
+        }
+        buffer.release();
+        buffer = Buffer.EMPTY;
+        pos = 0;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Filters.java

@@ -0,0 +1,100 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.plugin.PluginType;
+import org.embulk.spi.ExecSession;
+import org.embulk.spi.Schema;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.FilterPlugin;
+
+public abstract class Filters
+{
+    private Filters() { }
+
+    public static List<PluginType> getPluginTypes(List<ConfigSource> configs)
+    {
+        ImmutableList.Builder<PluginType> builder = ImmutableList.builder();
+        for (ConfigSource config : configs) {
+            builder.add(config.get(PluginType.class, "type"));
+        }
+        return builder.build();
+    }
+
+    public static List<FilterPlugin> newFilterPluginsFromConfigSources(ExecSession exec, List<ConfigSource> configs)
+    {
+        return newFilterPlugins(exec, getPluginTypes(configs));
+    }
+
+    public static List<FilterPlugin> newFilterPlugins(ExecSession exec, List<PluginType> pluginTypes)
+    {
+        ImmutableList.Builder<FilterPlugin> builder = ImmutableList.builder();
+        for (PluginType pluginType : pluginTypes) {
+            builder.add(exec.newPlugin(FilterPlugin.class, pluginType));
+        }
+        return builder.build();
+    }
+
+    public interface Control
+    {
+        public void run(List<TaskSource> taskSources, List<Schema> filterSchemas);
+    }
+
+    public static void transaction(List<FilterPlugin> plugins, List<ConfigSource> configs,
+            Schema inputSchema, Filters.Control control)
+    {
+        new RecursiveControl(plugins, configs, control).transaction(inputSchema);
+    }
+
+    public static PageOutput open(List<FilterPlugin> plugins, List<TaskSource> taskSources,
+            List<Schema> filterSchemas, PageOutput output)
+    {
+        PageOutput out = output;
+        int pos = plugins.size() - 1;
+        while (pos >= 0) {
+            out = plugins.get(pos).open(taskSources.get(pos), filterSchemas.get(pos), filterSchemas.get(pos + 1), out);
+            pos--;
+        }
+        return out;
+    }
+
+    private static class RecursiveControl
+    {
+        private final List<FilterPlugin> plugins;
+        private final List<ConfigSource> configs;
+        private final Filters.Control finalControl;
+        private final ImmutableList.Builder<TaskSource> taskSources;
+        private final ImmutableList.Builder<Schema> filterSchemas;
+        private int pos;
+
+        RecursiveControl(List<FilterPlugin> plugins, List<ConfigSource> configs,
+                Filters.Control finalControl)
+        {
+            this.plugins = plugins;
+            this.configs = configs;
+            this.finalControl = finalControl;
+            this.taskSources = ImmutableList.builder();
+            this.filterSchemas = ImmutableList.builder();
+        }
+
+        public void transaction(Schema inputSchema)
+        {
+            filterSchemas.add(inputSchema);
+            if (pos < plugins.size()) {
+                plugins.get(pos).transaction(configs.get(pos), inputSchema, new FilterPlugin.Control() {
+                    public void run(TaskSource taskSource, Schema outputSchema)
+                    {
+                        taskSources.add(taskSource);
+                        pos++;
+                        transaction(outputSchema);
+                    }
+                });
+            } else {
+                finalControl.run(taskSources.build(), filterSchemas.build());
+            }
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java

@@ -0,0 +1,190 @@
+package org.embulk.spi.util;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.io.InputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.BufferAllocator;
+
+public class InputStreamFileInput
+        implements FileInput
+{
+    public interface Provider extends Closeable
+    {
+        public InputStream openNext() throws IOException;
+
+        public void close() throws IOException;
+    }
+
+    public interface Opener
+    {
+        public InputStream open() throws IOException;
+    }
+
+    public static class IteratorProvider implements Provider
+    {
+        private Iterator<InputStream> iterator;
+
+        public IteratorProvider(Iterable<InputStream> iterable)
+        {
+            this.iterator = iterable.iterator();
+        }
+
+        public IteratorProvider(Iterator<InputStream> iterator)
+        {
+            this.iterator = iterator;
+        }
+
+        @Override
+        public InputStream openNext() throws IOException
+        {
+            if (!iterator.hasNext()) {
+                return null;
+            }
+            return iterator.next();
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            while (iterator.hasNext()) {
+                iterator.next().close();
+            }
+        }
+    }
+
+    private static class OpenerProvider implements Provider
+    {
+        private Opener opener;
+
+        public OpenerProvider(Opener opener)
+        {
+            this.opener = opener;
+        }
+
+        @Override
+        public InputStream openNext() throws IOException
+        {
+            if (opener == null) {
+                return null;
+            }
+            InputStream stream = opener.open();
+            opener = null;
+            return stream;
+        }
+
+        @Override
+        public void close() throws IOException
+        { }
+    }
+
+    private static class InputStreamProvider implements Provider
+    {
+        private InputStream input;
+
+        public InputStreamProvider(InputStream input)
+        {
+            this.input = input;
+        }
+
+        @Override
+        public InputStream openNext() throws IOException
+        {
+            if (input == null) {
+                return null;
+            }
+            InputStream ret = input;
+            input = null;
+            return ret;
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            if (input != null) {
+                input.close();
+                input = null;
+            }
+        }
+    }
+
+    private final BufferAllocator allocator;
+    private final Provider provider;
+    private InputStream current;
+
+    public InputStreamFileInput(BufferAllocator allocator, Provider provider)
+    {
+        this.allocator = allocator;
+        this.provider = provider;
+        this.current = null;
+    }
+
+    public InputStreamFileInput(BufferAllocator allocator, Opener opener)
+    {
+        this(allocator, new OpenerProvider(opener));
+    }
+
+    public InputStreamFileInput(BufferAllocator allocator, InputStream openedStream)
+    {
+        this(allocator, new InputStreamProvider(openedStream));
+    }
+
+    public Buffer poll()
+    {
+        // TODO check current != null and throw Illegal State - file is not opened
+        if (current == null) {
+            throw new IllegalStateException("nextFile() must be called before poll()");
+        }
+        Buffer buffer = allocator.allocate();
+        try {
+            int n = current.read(buffer.array(), buffer.offset(), buffer.capacity());
+            if (n < 0) {
+                return null;
+            }
+            buffer.limit(n);
+            Buffer b = buffer;
+            buffer = null;
+            return b;
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        } finally {
+            if (buffer != null) {
+                buffer.release();
+                buffer = null;
+            }
+        }
+    }
+
+    public boolean nextFile()
+    {
+        try {
+            if (current != null) {
+                current.close();
+                current = null;
+            }
+            current = provider.openNext();
+            return current != null;
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            try {
+                if (current != null) {
+                    current.close();
+                    current = null;
+                }
+            } finally {
+                provider.close();
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java

@@ -0,0 +1,25 @@
+package org.embulk.spi.util;
+
+import java.io.InputStream;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.BufferAllocator;
+
+public abstract class InputStreamTransactionalFileInput
+        extends InputStreamFileInput
+        implements TransactionalFileInput
+{
+    public InputStreamTransactionalFileInput(BufferAllocator allocator, Provider provider)
+    {
+        super(allocator, provider);
+    }
+
+    public InputStreamTransactionalFileInput(BufferAllocator allocator, Opener opener)
+    {
+        super(allocator, opener);
+    }
+
+    public InputStreamTransactionalFileInput(BufferAllocator allocator, InputStream openedStream)
+    {
+        super(allocator, openedStream);
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java

@@ -0,0 +1,65 @@
+package org.embulk.spi.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.joda.time.DateTimeZone;
+
+public class Inputs
+{
+    private static abstract class AbstractPollIterator <E>
+            implements Iterator<E>
+    {
+        private E next;
+
+        protected abstract E poll();
+
+        @Override
+        public boolean hasNext()
+        {
+            if (next != null) {
+                return true;
+            } else {
+                next = poll();
+                return next != null;
+            }
+        }
+
+        @Override
+        public E next()
+        {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            E l = next;
+            next = null;
+            return l;
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    public static Iterable<Buffer> each(final FileInput input)
+    {
+        return new Iterable<Buffer>() {
+            public Iterator<Buffer> iterator()
+            {
+                return new AbstractPollIterator<Buffer>() {
+                    public Buffer poll()
+                    {
+                        return input.poll();
+                    }
+                };
+            }
+        };
+    }
+}