embulk 0.1.0

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java

@@ -0,0 +1,111 @@
+package org.embulk.spi.util;
+
+import java.util.Arrays;
+import java.util.Iterator;
+import java.io.InputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.BufferAllocator;
+
+public class InputStreamFileInput
+        implements FileInput
+{
+    public interface Provider extends Closeable
+    {
+        public InputStream openNext() throws IOException;
+
+        public void close() throws IOException;
+    }
+
+    public static class IteratorProvider implements Provider {
+        private Iterator<InputStream> iterator;
+
+        public IteratorProvider(Iterable<InputStream> iterable)
+        {
+            this.iterator = iterable.iterator();
+        }
+
+        public IteratorProvider(Iterator<InputStream> iterator)
+        {
+            this.iterator = iterator;
+        }
+
+        public InputStream openNext() throws IOException
+        {
+            if (!iterator.hasNext()) {
+                return null;
+            }
+            return iterator.next();
+        }
+
+        @Override
+        public void close() throws IOException
+        {
+            while (iterator.hasNext()) {
+                iterator.next().close();
+            }
+        }
+    }
+
+    private final BufferAllocator allocator;
+    private final Provider provider;
+    private InputStream current;
+
+    public InputStreamFileInput(BufferAllocator allocator, Provider provider)
+    {
+        this.allocator = allocator;
+        this.provider = provider;
+        this.current = null;
+    }
+
+    public Buffer poll()
+    {
+        // TODO check current != null and throw Illegal State - file is not opened
+        if (current == null) {
+            throw new IllegalStateException("openNext must be called before poll()");
+        }
+        Buffer buffer = allocator.allocate();
+        try {
+            int n = current.read(buffer.array(), buffer.offset(), buffer.capacity());
+            if (n < 0) {
+                return null;
+            }
+            buffer.limit(n);
+            Buffer b = buffer;
+            buffer = null;
+            return b;
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        } finally {
+            if (buffer != null) {
+                buffer.release();
+                buffer = null;
+            }
+        }
+    }
+
+    public boolean nextFile()
+    {
+        try {
+            if (current != null) {
+                current.close();
+                current = null;
+            }
+            current = provider.openNext();
+            return current != null;
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            provider.close();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java

@@ -0,0 +1,74 @@
+package org.embulk.spi.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInput;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.joda.time.DateTimeZone;
+
+public class Inputs
+{
+    private static abstract class AbstractPollIterator <E>
+            implements Iterator<E>
+    {
+        private E next;
+
+        protected abstract E poll();
+
+        @Override
+        public boolean hasNext()
+        {
+            if (next != null) {
+                return true;
+            } else {
+                next = poll();
+                return next != null;
+            }
+        }
+
+        @Override
+        public E next()
+        {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            E l = next;
+            next = null;
+            return l;
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+
+    public static Iterable<Buffer> each(final FileInput input)
+    {
+        return new Iterable<Buffer>() {
+            public Iterator<Buffer> iterator()
+            {
+                return new AbstractPollIterator<Buffer>() {
+                    public Buffer poll()
+                    {
+                        return input.poll();
+                    }
+                };
+            }
+        };
+    }
+
+    public static String formatPath(String pathFormat)
+    {
+        Timestamp timestamp = Exec.session().getTransactionTime();
+        DateTimeZone timezone = Exec.session().getTransactionTimeZone();
+        // newTimestampFormatter (eventually calls org.jruby.util.RubyDateFormat.<init> doesn't throw exceptions
+        TimestampFormatter formatter = Exec.session().newTimestampFormatter(pathFormat, timezone);
+        return formatter.format(timestamp);  // TimestampFormatter.format doesn't throw exceptions
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java

@@ -0,0 +1,118 @@
+package org.embulk.spi.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.io.Reader;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.FileInput;
+
+public class LineDecoder
+        implements AutoCloseable, Iterable<String>
+{
+    // TODO optimize
+
+    public static interface DecoderTask
+            extends Task
+    {
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public Charset getCharset();
+
+        @Config("newline")
+        @ConfigDefault("\"CRLF\"")
+        public Newline getNewline();
+    }
+
+    private final FileInputInputStream inputStream;
+    private final BufferedReader reader;
+
+    public LineDecoder(FileInput in, DecoderTask task)
+    {
+        CharsetDecoder decoder = task.getCharset()
+            .newDecoder()
+            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
+            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
+        this.inputStream = new FileInputInputStream(in);
+        this.reader = new BufferedReader(new InputStreamReader(inputStream, decoder));
+    }
+
+    public boolean nextFile()
+    {
+        return inputStream.nextFile();
+    }
+
+    public String poll()
+    {
+        try {
+            return reader.readLine();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            reader.close();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public Iterator<String> iterator()
+    {
+        return new Ite(this);
+    }
+
+    private String nextLine;
+
+    private static class Ite
+            implements Iterator<String>
+    {
+        private LineDecoder self;
+
+        public Ite(LineDecoder self)
+        {
+            // TODO non-static inner class causes a problem with JRuby
+            this.self = self;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            if (self.nextLine != null) {
+                return true;
+            } else {
+                self.nextLine = self.poll();
+                return self.nextLine != null;
+            }
+        }
+
+        @Override
+        public String next()
+        {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            String l = self.nextLine;
+            self.nextLine = null;
+            return l;
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -0,0 +1,109 @@
+package org.embulk.spi.util;
+
+import java.io.Writer;
+import java.io.OutputStreamWriter;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import com.fasterxml.jackson.annotation.JacksonInject;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
+
+public class LineEncoder
+        implements AutoCloseable
+{
+    // TODO optimize
+
+    public interface EncoderTask
+            extends Task
+    {
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public Charset getCharset();
+
+        @Config("newline")
+        @ConfigDefault("\"CRLF\"")
+        public Newline getNewline();
+
+        @JacksonInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    private final String newline;
+    private final FileOutputOutputStream outputStream;
+    private final Writer writer;
+
+    public LineEncoder(FileOutput out, EncoderTask task)
+    {
+        CharsetEncoder encoder = task.getCharset()
+            .newEncoder()
+            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
+            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
+        this.newline = task.getNewline().getString();
+        this.outputStream = new FileOutputOutputStream(out, task.getBufferAllocator());
+        this.writer = new OutputStreamWriter(outputStream, encoder);
+    }
+
+    public void addNewLine()
+    {
+        try {
+            writer.append(newline);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void addLine(String line)
+    {
+        try {
+            writer.append(line);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+        addNewLine();
+    }
+
+    public void addText(String text)
+    {
+        try {
+            writer.append(text);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void nextFile()
+    {
+        try {
+            writer.flush();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+        outputStream.nextFile();
+    }
+
+    public void finish()
+    {
+        close();   // flush all remaining buffer in writer
+        outputStream.finish();
+    }
+
+    @Override
+    public void close()
+    {
+        try {
+            writer.close();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java

@@ -0,0 +1,52 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+
+public class ListFileInput
+        implements FileInput
+{
+    private Iterator<? extends Iterable<Buffer>> files;
+    private Iterator<Buffer> currentBuffers;
+
+    public ListFileInput(Iterable<? extends Iterable<Buffer>> files)
+    {
+        this.files = files.iterator();
+    }
+
+    public boolean nextFile()
+    {
+        if (!files.hasNext()) {
+            return false;
+        }
+        currentBuffers = files.next().iterator();
+        return true;
+    }
+
+    public Buffer poll()
+    {
+        if (currentBuffers == null) {
+            throw new IllegalStateException("FileInput.nextFile is not called");
+        }
+        if (!currentBuffers.hasNext()) {
+            return null;
+        }
+        return currentBuffers.next();
+    }
+
+    public void close()
+    {
+        do {
+            while (true) {
+                Buffer b = poll();
+                if (b == null) {
+                    break;
+                }
+                b.release();
+            }
+        } while (nextFile());
+    }
+}