embulk 0.8.36-java → 0.10.25-java

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamFileInput.java

@@ -1,189 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Arrays;
-import java.util.Iterator;
-import java.io.InputStream;
-import java.io.Closeable;
-import java.io.IOException;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.BufferAllocator;
-
-public class InputStreamFileInput
-        implements FileInput
-{
-    public interface Provider extends Closeable
-    {
-        public InputStream openNext() throws IOException;
-
-        public void close() throws IOException;
-    }
-
-    public interface Opener
-    {
-        public InputStream open() throws IOException;
-    }
-
-    public static class IteratorProvider implements Provider
-    {
-        private Iterator<InputStream> iterator;
-
-        public IteratorProvider(Iterable<InputStream> iterable)
-        {
-            this.iterator = iterable.iterator();
-        }
-
-        public IteratorProvider(Iterator<InputStream> iterator)
-        {
-            this.iterator = iterator;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (!iterator.hasNext()) {
-                return null;
-            }
-            return iterator.next();
-        }
-
-        @Override
-        public void close() throws IOException
-        {
-            while (iterator.hasNext()) {
-                iterator.next().close();
-            }
-        }
-    }
-
-    private static class OpenerProvider implements Provider
-    {
-        private Opener opener;
-
-        public OpenerProvider(Opener opener)
-        {
-            this.opener = opener;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (opener == null) {
-                return null;
-            }
-            InputStream stream = opener.open();
-            opener = null;
-            return stream;
-        }
-
-        @Override
-        public void close() throws IOException
-        { }
-    }
-
-    private static class InputStreamProvider implements Provider
-    {
-        private InputStream input;
-
-        public InputStreamProvider(InputStream input)
-        {
-            this.input = input;
-        }
-
-        @Override
-        public InputStream openNext() throws IOException
-        {
-            if (input == null) {
-                return null;
-            }
-            InputStream ret = input;
-            input = null;
-            return ret;
-        }
-
-        @Override
-        public void close() throws IOException
-        {
-            if (input != null) {
-                input.close();
-                input = null;
-            }
-        }
-    }
-
-    private final BufferAllocator allocator;
-    private final Provider provider;
-    private InputStream current;
-
-    public InputStreamFileInput(BufferAllocator allocator, Provider provider)
-    {
-        this.allocator = allocator;
-        this.provider = provider;
-        this.current = null;
-    }
-
-    public InputStreamFileInput(BufferAllocator allocator, Opener opener)
-    {
-        this(allocator, new OpenerProvider(opener));
-    }
-
-    public InputStreamFileInput(BufferAllocator allocator, InputStream openedStream)
-    {
-        this(allocator, new InputStreamProvider(openedStream));
-    }
-
-    public Buffer poll()
-    {
-        if (current == null) {
-            throw new IllegalStateException("nextFile() must be called before poll()");
-        }
-        Buffer buffer = allocator.allocate();
-        try {
-            int n = current.read(buffer.array(), buffer.offset(), buffer.capacity());
-            if (n < 0) {
-                return null;
-            }
-            buffer.limit(n);
-            Buffer b = buffer;
-            buffer = null;
-            return b;
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        } finally {
-            if (buffer != null) {
-                buffer.release();
-                buffer = null;
-            }
-        }
-    }
-
-    public boolean nextFile()
-    {
-        try {
-            if (current != null) {
-                current.close();
-                current = null;
-            }
-            current = provider.openNext();
-            return current != null;
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void close()
-    {
-        try {
-            try {
-                if (current != null) {
-                    current.close();
-                    current = null;
-                }
-            } finally {
-                provider.close();
-            }
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/InputStreamTransactionalFileInput.java

@@ -1,25 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.InputStream;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.BufferAllocator;
-
-public abstract class InputStreamTransactionalFileInput
-        extends InputStreamFileInput
-        implements TransactionalFileInput
-{
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, Provider provider)
-    {
-        super(allocator, provider);
-    }
-
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, Opener opener)
-    {
-        super(allocator, opener);
-    }
-
-    public InputStreamTransactionalFileInput(BufferAllocator allocator, InputStream openedStream)
-    {
-        super(allocator, openedStream);
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/Inputs.java

@@ -1,65 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInput;
-import org.embulk.spi.Buffer;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.joda.time.DateTimeZone;
-
-public class Inputs
-{
-    private static abstract class AbstractPollIterator <E>
-            implements Iterator<E>
-    {
-        private E next;
-
-        protected abstract E poll();
-
-        @Override
-        public boolean hasNext()
-        {
-            if (next != null) {
-                return true;
-            } else {
-                next = poll();
-                return next != null;
-            }
-        }
-
-        @Override
-        public E next()
-        {
-            if (!hasNext()) {
-                throw new NoSuchElementException();
-            }
-            E l = next;
-            next = null;
-            return l;
-        }
-
-        @Override
-        public void remove()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-
-    public static Iterable<Buffer> each(final FileInput input)
-    {
-        return new Iterable<Buffer>() {
-            public Iterator<Buffer> iterator()
-            {
-                return new AbstractPollIterator<Buffer>() {
-                    public Buffer poll()
-                    {
-                        return input.poll();
-                    }
-                };
-            }
-        };
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/LineDecoder.java

@@ -1,157 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Iterator;
-import java.util.NoSuchElementException;
-import java.io.Reader;
-import java.io.BufferedReader;
-import java.io.InputStreamReader;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
-import java.nio.charset.CodingErrorAction;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.spi.FileInput;
-import static java.nio.charset.StandardCharsets.UTF_8;
-
-public class LineDecoder
-        implements AutoCloseable, Iterable<String>
-{
-    // TODO optimize
-
-    public static interface DecoderTask
-            extends Task
-    {
-        @Config("charset")
-        @ConfigDefault("\"utf-8\"")
-        public Charset getCharset();
-
-        @Config("newline")
-        @ConfigDefault("\"CRLF\"")
-        public Newline getNewline();
-    }
-
-    private final FileInputInputStream inputStream;
-    private final BufferedReader reader;
-    private final Charset charset;
-
-    public LineDecoder(FileInput in, DecoderTask task)
-    {
-        this.charset = task.getCharset();
-        CharsetDecoder decoder = charset
-            .newDecoder()
-            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
-            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
-        this.inputStream = new FileInputInputStream(in);
-        this.reader = new BufferedReader(new InputStreamReader(inputStream, decoder));
-    }
-
-    public boolean nextFile()
-    {
-        boolean has = inputStream.nextFile();
-        if (has && charset.equals(UTF_8)) {
-            skipBom();
-        }
-        return has;
-    }
-
-    private void skipBom()
-    {
-        boolean skip = false;
-        try {
-            if (charset.equals(UTF_8)) {
-                reader.mark(3);
-                int firstChar = reader.read();
-                if (firstChar == 0xFEFF) {
-                    // skip BOM bytes
-                    skip = true;
-                }
-            }
-        }
-        catch (IOException ex) {
-        }
-        finally {
-            if (skip) {
-                // firstChar is skipped
-            }
-            else {
-                // rollback to the marked position
-                try {
-                    reader.reset();
-                }
-                catch (IOException ex) {
-                    // unexpected
-                    throw new RuntimeException(ex);
-                }
-            }
-        }
-    }
-
-    public String poll()
-    {
-        try {
-            return reader.readLine();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void close()
-    {
-        try {
-            reader.close();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public Iterator<String> iterator()
-    {
-        return new Ite(this);
-    }
-
-    private String nextLine;
-
-    private static class Ite
-            implements Iterator<String>
-    {
-        private LineDecoder self;
-
-        public Ite(LineDecoder self)
-        {
-            // TODO non-static inner class causes a problem with JRuby
-            this.self = self;
-        }
-
-        @Override
-        public boolean hasNext()
-        {
-            if (self.nextLine != null) {
-                return true;
-            } else {
-                self.nextLine = self.poll();
-                return self.nextLine != null;
-            }
-        }
-
-        @Override
-        public String next()
-        {
-            if (!hasNext()) {
-                throw new NoSuchElementException();
-            }
-            String l = self.nextLine;
-            self.nextLine = null;
-            return l;
-        }
-
-        @Override
-        public void remove()
-        {
-            throw new UnsupportedOperationException();
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/LineEncoder.java

@@ -1,123 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.Writer;
-import java.io.BufferedWriter;
-import java.io.OutputStreamWriter;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.nio.charset.CharsetEncoder;
-import java.nio.charset.CodingErrorAction;
-import org.embulk.config.Task;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigDefault;
-import org.embulk.spi.FileOutput;
-import org.embulk.spi.BufferAllocator;
-
-public class LineEncoder
-        implements AutoCloseable
-{
-    // TODO optimize
-
-    public interface EncoderTask
-            extends Task
-    {
-        @Config("charset")
-        @ConfigDefault("\"utf-8\"")
-        public Charset getCharset();
-
-        @Config("newline")
-        @ConfigDefault("\"CRLF\"")
-        public Newline getNewline();
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    private final String newline;
-    private final FileOutput underlyingFileOutput;
-    private final FileOutputOutputStream outputStream;
-    private Writer writer;
-
-    public LineEncoder(FileOutput out, EncoderTask task)
-    {
-        CharsetEncoder encoder = task.getCharset()
-            .newEncoder()
-            .onMalformedInput(CodingErrorAction.REPLACE)  // TODO configurable?
-            .onUnmappableCharacter(CodingErrorAction.REPLACE);  // TODO configurable?
-        this.newline = task.getNewline().getString();
-        this.underlyingFileOutput = out;
-        this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH);
-        this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32*1024);
-    }
-
-    public void addNewLine()
-    {
-        try {
-            writer.append(newline);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void addLine(String line)
-    {
-        try {
-            writer.append(line);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-        addNewLine();
-    }
-
-    public void addText(String text)
-    {
-        try {
-            writer.append(text);
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-
-    public void nextFile()
-    {
-        try {
-            writer.flush();
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-        outputStream.nextFile();
-    }
-
-    public void finish()
-    {
-        try {
-            if (writer != null) {
-                writer.close();  // FLUSH_FINISH
-                writer = null;
-                // underlyingFileOutput.finish() is already called by close() because CloseMode is FLUSH_FINISH
-            }
-        } catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    @Override
-    public void close()
-    {
-        try {
-            if (writer != null) {
-                writer.close();  // FLUSH_FINISH
-                writer = null;
-            }
-            underlyingFileOutput.close();  // this is necessary because CloseMode is not FLUSH_FINISH_CLOSE
-        } catch (IOException ex) {
-            // unexpected
-            throw new RuntimeException(ex);
-        }
-    }
-}