embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java

@@ -0,0 +1,118 @@
+package org.embulk.spi.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.io.Reader;
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetDecoder;
+import java.nio.charset.CodingErrorAction;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.FileInput;
+
+public class LineDecoder
+        implements AutoCloseable, Iterable<String>
+{
+    // TODO optimize
+
+    public static interface DecoderTask
+            extends Task
+    {
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public Charset getCharset();
+
+        @Config("newline")
+        @ConfigDefault("\"CRLF\"")
+        public Newline getNewline();
+    }
+
+    private final FileInputInputStream inputStream;
+    private final BufferedReader reader;
+
+    public LineDecoder(FileInput in, DecoderTask task)
+    {
+        CharsetDecoder decoder = task.getCharset()
+            .newDecoder()
+            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
+            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
+        this.inputStream = new FileInputInputStream(in);
+        this.reader = new BufferedReader(new InputStreamReader(inputStream, decoder));
+    }
+
+    public boolean nextFile()
+    {
+        return inputStream.nextFile();
+    }
+
+    public String poll()
+    {
+        try {
+            return reader.readLine();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            reader.close();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public Iterator<String> iterator()
+    {
+        return new Ite(this);
+    }
+
+    private String nextLine;
+
+    private static class Ite
+            implements Iterator<String>
+    {
+        private LineDecoder self;
+
+        public Ite(LineDecoder self)
+        {
+            // TODO non-static inner class causes a problem with JRuby
+            this.self = self;
+        }
+
+        @Override
+        public boolean hasNext()
+        {
+            if (self.nextLine != null) {
+                return true;
+            } else {
+                self.nextLine = self.poll();
+                return self.nextLine != null;
+            }
+        }
+
+        @Override
+        public String next()
+        {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            String l = self.nextLine;
+            self.nextLine = null;
+            return l;
+        }
+
+        @Override
+        public void remove()
+        {
+            throw new UnsupportedOperationException();
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -0,0 +1,123 @@
+package org.embulk.spi.util;
+
+import java.io.Writer;
+import java.io.BufferedWriter;
+import java.io.OutputStreamWriter;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.CharsetEncoder;
+import java.nio.charset.CodingErrorAction;
+import org.embulk.config.Task;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.FileOutput;
+import org.embulk.spi.BufferAllocator;
+
+public class LineEncoder
+        implements AutoCloseable
+{
+    // TODO optimize
+
+    public interface EncoderTask
+            extends Task
+    {
+        @Config("charset")
+        @ConfigDefault("\"utf-8\"")
+        public Charset getCharset();
+
+        @Config("newline")
+        @ConfigDefault("\"CRLF\"")
+        public Newline getNewline();
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    private final String newline;
+    private final FileOutput underlyingFileOutput;
+    private final FileOutputOutputStream outputStream;
+    private Writer writer;
+
+    public LineEncoder(FileOutput out, EncoderTask task)
+    {
+        CharsetEncoder encoder = task.getCharset()
+            .newEncoder()
+            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
+            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
+        this.newline = task.getNewline().getString();
+        this.underlyingFileOutput = out;
+        this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH);
+        this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32*1024);
+    }
+
+    public void addNewLine()
+    {
+        try {
+            writer.append(newline);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void addLine(String line)
+    {
+        try {
+            writer.append(line);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+        addNewLine();
+    }
+
+    public void addText(String text)
+    {
+        try {
+            writer.append(text);
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void nextFile()
+    {
+        try {
+            writer.flush();
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+        outputStream.nextFile();
+    }
+
+    public void finish()
+    {
+        try {
+            if (writer != null) {
+                writer.close();  // FLUSH_FINISH
+                writer = null;
+                // underlyingFileOutput.finish() is already called by close() because CloseMode is FLUSH_FINISH
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    @Override
+    public void close()
+    {
+        try {
+            if (writer != null) {
+                writer.close();  // FLUSH_FINISH
+                writer = null;
+            }
+            underlyingFileOutput.close();  // this is necessary because CloseMode is not FLUSH_FINISH_CLOSE
+        } catch (IOException ex) {
+            // unexpected
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java

@@ -0,0 +1,52 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Iterator;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileInput;
+
+public class ListFileInput
+        implements FileInput
+{
+    private Iterator<? extends Iterable<Buffer>> files;
+    private Iterator<Buffer> currentBuffers;
+
+    public ListFileInput(Iterable<? extends Iterable<Buffer>> files)
+    {
+        this.files = files.iterator();
+    }
+
+    public boolean nextFile()
+    {
+        if (!files.hasNext()) {
+            return false;
+        }
+        currentBuffers = files.next().iterator();
+        return true;
+    }
+
+    public Buffer poll()
+    {
+        if (currentBuffers == null) {
+            throw new IllegalStateException("FileInput.nextFile is not called");
+        }
+        if (!currentBuffers.hasNext()) {
+            return null;
+        }
+        return currentBuffers.next();
+    }
+
+    public void close()
+    {
+        do {
+            while (true) {
+                Buffer b = poll();
+                if (b == null) {
+                    break;
+                }
+                b.release();
+            }
+        } while (nextFile());
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java

@@ -0,0 +1,38 @@
+package org.embulk.spi.util;
+
+public enum Newline
+{
+    CRLF("\r\n"),
+    LF("\n"),
+    CR("\r");
+
+    private final String string;
+    private final char firstCharCode;
+    private final char secondCharCode;
+
+    private Newline(String string)
+    {
+        this.string = string;
+        this.firstCharCode = string.charAt(0);
+        if (string.length() > 1) {
+            this.secondCharCode = string.charAt(1);
+        } else {
+            this.secondCharCode = 0;
+        }
+    }
+
+    public String getString()
+    {
+        return string;
+    }
+
+    public char getFirstCharCode()
+    {
+        return firstCharCode;
+    }
+
+    public char getSecondCharCode()
+    {
+        return secondCharCode;
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/OutputStreamFileOutput.java

@@ -0,0 +1,88 @@
+package org.embulk.spi.util;
+
+import java.io.OutputStream;
+import java.io.Closeable;
+import java.io.IOException;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.FileOutput;
+
+public class OutputStreamFileOutput
+        implements FileOutput
+{
+    public interface Provider extends Closeable
+    {
+        public OutputStream openNext() throws IOException;
+
+        public void finish() throws IOException;
+
+        public void close() throws IOException;
+    }
+
+    private final Provider provider;
+    private OutputStream current;
+
+    public OutputStreamFileOutput(Provider provider)
+    {
+        this.provider = provider;
+        this.current = null;
+    }
+
+    public void nextFile()
+    {
+        closeCurrent();
+        try {
+            current = provider.openNext();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void add(Buffer buffer)
+    {
+        if (current == null) {
+            throw new IllegalStateException("nextFile() must be called before poll()");
+        }
+        try {
+            current.write(buffer.array(), buffer.offset(), buffer.limit());
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        } finally {
+            buffer.release();
+        }
+    }
+
+    public void finish()
+    {
+        closeCurrent();
+        try {
+            provider.finish();
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public void close()
+    {
+        try {
+            closeCurrent();
+        } finally {
+            try {
+                provider.close();
+            } catch (IOException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+
+    private void closeCurrent()
+    {
+        try {
+            if (current != null) {
+                current.close();
+                current = null;
+            }
+        } catch (IOException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/PagePrinter.java

@@ -0,0 +1,102 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.ArrayList;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.Schema;
+import org.embulk.spi.Column;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.type.TimestampType;
+
+public class PagePrinter
+{
+    private final Schema schema;
+    private final TimestampFormatter[] timestampFormatters;
+    private final ArrayList<String> record;
+
+    public PagePrinter(Schema schema, TimestampFormatter.FormatterTask task)
+    {
+        this.schema = schema;
+        this.timestampFormatters = new TimestampFormatter[schema.getColumnCount()];
+        for (int i=0; i < timestampFormatters.length; i++) {
+            if (schema.getColumnType(i) instanceof TimestampType) {
+                TimestampType type = (TimestampType) schema.getColumnType(i);
+                timestampFormatters[i] = new TimestampFormatter(type.getFormat(), task);
+            }
+        }
+
+        this.record = new ArrayList<String>(schema.getColumnCount());
+        for (int i=0; i < schema.getColumnCount(); i++) {
+            record.add("");
+        }
+    }
+
+    public String printRecord(PageReader reader, String delimiter)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (Column c : schema.getColumns()) {
+            if (c.getIndex() != 0) {
+                sb.append(delimiter);
+            }
+            sb.append(printColumn(reader, c));
+        }
+        return sb.toString();
+    }
+
+    public List<String> printRecord(PageReader reader)
+    {
+        for (Column c : schema.getColumns()) {
+            record.set(c.getIndex(), printColumn(reader, c));
+        }
+        return record;
+    }
+
+    public String printColumn(PageReader reader, Column column)
+    {
+        if (reader.isNull(column)) {
+            return "";
+        }
+        ToStringColumnVisitor visitor = new ToStringColumnVisitor(reader);
+        column.visit(visitor);
+        return visitor.string;
+    }
+
+    private class ToStringColumnVisitor
+            implements ColumnVisitor
+    {
+        private final PageReader reader;
+        String string = "";
+
+        public ToStringColumnVisitor(PageReader reader)
+        {
+            this.reader = reader;
+        }
+
+        public void booleanColumn(Column column)
+        {
+            string = Boolean.toString(reader.getBoolean(column));
+        }
+
+        public void longColumn(Column column)
+        {
+            string = Long.toString(reader.getLong(column));
+        }
+
+        public void doubleColumn(Column column)
+        {
+            string = Double.toString(reader.getDouble(column));
+        }
+
+        public void stringColumn(Column column)
+        {
+            string = reader.getString(column);
+        }
+
+        public void timestampColumn(Column column)
+        {
+            string = timestampFormatters[column.getIndex()].format(reader.getTimestamp(column));
+        }
+    }
+}