embulk 0.8.37-java → 0.10.26-java

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java

@@ -1,25 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.InputStream;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.BufferAllocator;
-
-public abstract class InputStreamTransactionalFileInput
-        extends InputStreamFileInput
-        implements TransactionalFileInput
-{
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, Provider provider)
-    {
-        super(allocator, provider);
-    }
-
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, Opener opener)
-    {
-        super(allocator, opener);
-    }
-
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, InputStream openedStream)
-    {
-        super(allocator, openedStream);
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java

@@ -1,65 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.joda.time.DateTimeZone;
-
-public class Inputs
-{
-    private static abstract class AbstractPollIterator <E>
-            implements Iterator<E>
-    {
-        private E next;
-
-        protected abstract E poll();
-
-        @Override
-        public boolean hasNext()
-        {
-            if (next != null) {
-                return true;
-            } else {
-                next = poll();
-                return next != null;
-            }
-        }
-
-        @Override
-        public E next()
-        {
-            if (!hasNext()) {
-                throw new NoSuchElementException();
-            }
-            E l = next;
-            next = null;
-            return l;
-        }
-
-        @Override
-        public void remove()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    public static Iterable<Buffer> each(final FileInput input)
-    {
-        return new Iterable<Buffer>() {
-            public Iterator<Buffer> iterator()
-            {
-                return new AbstractPollIterator<Buffer>() {
-                    public Buffer poll()
-                    {
-                        return input.poll();
-                    }
-                };
-            }
-        };
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java

@@ -1,157 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.io.Reader;
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.spi.FileInput;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-public class LineDecoder
-        implements AutoCloseable, Iterable<String>
-{
-    // TODO optimize
-
-    public static interface DecoderTask
-            extends Task
-    {
-        @Config("charset")
-        @ConfigDefault("\"utf-8\"")
-        public Charset getCharset();
-
-        @Config("newline")
-        @ConfigDefault("\"CRLF\"")
-        public Newline getNewline();
-    }
-
-    private final FileInputInputStream inputStream;
-    private final BufferedReader reader;
-    private final Charset charset;
-
-    public LineDecoder(FileInput in, DecoderTask task)
-    {
-        this.charset = task.getCharset();
-        CharsetDecoder decoder = charset
-            .newDecoder()
-            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
-            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
-        this.inputStream = new FileInputInputStream(in);
-        this.reader = new BufferedReader(new InputStreamReader(inputStream, decoder));
-    }
-
-    public boolean nextFile()
-    {
-        boolean has = inputStream.nextFile();
-        if (has && charset.equals(UTF_8)) {
-            skipBom();
-        }
-        return has;
-    }
-
-    private void skipBom()
-    {
-        boolean skip = false;
-        try {
-            if (charset.equals(UTF_8)) {
-                reader.mark(3);
-                int firstChar = reader.read();
-                if (firstChar == 0xFEFF) {
-                    // skip BOM bytes
-                    skip = true;
-                }
-            }
-        }
-        catch (IOException ex) {
-        }
-        finally {
-            if (skip) {
-                // firstChar is skipped
-            }
-            else {
-                // rollback to the marked position
-                try {
-                    reader.reset();
-                }
-                catch (IOException ex) {
-                    // unexpected
-                    throw new RuntimeException(ex);
-                }
-            }
-        }
-    }
-
-    public String poll()
-    {
-        try {
-            return reader.readLine();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void close()
-    {
-        try {
-            reader.close();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public Iterator<String> iterator()
-    {
-        return new Ite(this);
-    }
-
-    private String nextLine;
-
-    private static class Ite
-            implements Iterator<String>
-    {
-        private LineDecoder self;
-
-        public Ite(LineDecoder self)
-        {
-            // TODO non-static inner class causes a problem with JRuby
-            this.self = self;
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            if (self.nextLine != null) {
-                return true;
-            } else {
-                self.nextLine = self.poll();
-                return self.nextLine != null;
-            }
-        }
-
-        @Override
-        public String next()
-        {
-            if (!hasNext()) {
-                throw new NoSuchElementException();
-            }
-            String l = self.nextLine;
-            self.nextLine = null;
-            return l;
-        }
-
-        @Override
-        public void remove()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -1,123 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.Writer;
-import java.io.BufferedWriter;
-import java.io.OutputStreamWriter;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CodingErrorAction;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigDefault;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.BufferAllocator;
-
-public class LineEncoder
-        implements AutoCloseable
-{
-    // TODO optimize
-
-    public interface EncoderTask
-            extends Task
-    {
-        @Config("charset")
-        @ConfigDefault("\"utf-8\"")
-        public Charset getCharset();
-
-        @Config("newline")
-        @ConfigDefault("\"CRLF\"")
-        public Newline getNewline();
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    private final String newline;
-    private final FileOutput underlyingFileOutput;
-    private final FileOutputOutputStream outputStream;
-    private Writer writer;
-
-    public LineEncoder(FileOutput out, EncoderTask task)
-    {
-        CharsetEncoder encoder = task.getCharset()
-            .newEncoder()
-            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
-            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
-        this.newline = task.getNewline().getString();
-        this.underlyingFileOutput = out;
-        this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH);
-        this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32*1024);
-    }
-
-    public void addNewLine()
-    {
-        try {
-            writer.append(newline);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void addLine(String line)
-    {
-        try {
-            writer.append(line);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-        addNewLine();
-    }
-
-    public void addText(String text)
-    {
-        try {
-            writer.append(text);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void nextFile()
-    {
-        try {
-            writer.flush();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-        outputStream.nextFile();
-    }
-
-    public void finish()
-    {
-        try {
-            if (writer != null) {
-                writer.close();  // FLUSH_FINISH
-                writer = null;
-                // underlyingFileOutput.finish() is already called by close() because CloseMode is FLUSH_FINISH
-            }
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        try {
-            if (writer != null) {
-                writer.close();  // FLUSH_FINISH
-                writer = null;
-            }
-            underlyingFileOutput.close();  // this is necessary because CloseMode is not FLUSH_FINISH_CLOSE
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/ListFileInput.java

@@ -1,52 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Iterator;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInput;
-
-public class ListFileInput
-        implements FileInput
-{
-    private Iterator<? extends Iterable<Buffer>> files;
-    private Iterator<Buffer> currentBuffers;
-
-    public ListFileInput(Iterable<? extends Iterable<Buffer>> files)
-    {
-        this.files = files.iterator();
-    }
-
-    public boolean nextFile()
-    {
-        if (!files.hasNext()) {
-            return false;
-        }
-        currentBuffers = files.next().iterator();
-        return true;
-    }
-
-    public Buffer poll()
-    {
-        if (currentBuffers == null) {
-            throw new IllegalStateException("FileInput.nextFile is not called");
-        }
-        if (!currentBuffers.hasNext()) {
-            return null;
-        }
-        return currentBuffers.next();
-    }
-
-    public void close()
-    {
-        do {
-            while (true) {
-                Buffer b = poll();
-                if (b == null) {
-                    break;
-                }
-                b.release();
-            }
-        } while (nextFile());
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Newline.java

@@ -1,38 +0,0 @@
-package org.embulk.spi.util;
-
-public enum Newline
-{
-    CRLF("\r\n"),
-    LF("\n"),
-    CR("\r");
-
-    private final String string;
-    private final char firstCharCode;
-    private final char secondCharCode;
-
-    private Newline(String string)
-    {
-        this.string = string;
-        this.firstCharCode = string.charAt(0);
-        if (string.length() > 1) {
-            this.secondCharCode = string.charAt(1);
-        } else {
-            this.secondCharCode = 0;
-        }
-    }
-
-    public String getString()
-    {
-        return string;
-    }
-
-    public char getFirstCharCode()
-    {
-        return firstCharCode;
-    }
-
-    public char getSecondCharCode()
-    {
-        return secondCharCode;
-    }
-}