embulk-input-ftp 0.1.0

data/src/main/java/org/embulk/input/ftp/BlockingTransfer.java

@@ -0,0 +1,267 @@
+package org.embulk.input.ftp;
+
+import java.util.concurrent.Future;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.io.IOException;
+import java.io.EOFException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.ReadableByteChannel;
+import java.nio.channels.WritableByteChannel;
+import com.google.common.base.Function;
+
+public class BlockingTransfer
+{
+    private final WriterChannel writerChannel;
+    private final ReaderChannel readerChannel;
+    private Future<?> transferCompletionFuture;
+
+    public static BlockingTransfer submit(ExecutorService executor,
+            Function<BlockingTransfer, Runnable> starterFactory)
+    {
+        BlockingTransfer transfer = new BlockingTransfer();
+        final Runnable starter = starterFactory.apply(transfer);
+        transfer.setTransferCompletionFuture(
+                executor.submit(new Callable<Void>() {
+                    public Void call() throws Exception
+                    {
+                        starter.run();
+                        return null;
+                    }
+                })
+            );
+        return transfer;
+    }
+
+    private BlockingTransfer()
+    {
+        this.writerChannel = new WriterChannel();
+        this.readerChannel = new ReaderChannel();
+    }
+
+    private void setTransferCompletionFuture(Future<?> future)
+    {
+        this.transferCompletionFuture = future;
+    }
+
+    public ReadableByteChannel getReaderChannel()
+    {
+        return readerChannel;
+    }
+
+    public WritableByteChannel getWriterChannel()
+    {
+        return writerChannel;
+    }
+
+    public void transferFailed(Throwable exception)
+    {
+        readerChannel.overwriteException(exception);
+    }
+
+    void waitForTransferCompletion() throws IOException
+    {
+        Future<?> f = transferCompletionFuture;
+        if (f != null) {
+            try {
+                f.get();
+            } catch (CancellationException ex) {
+                throw new InterruptedIOException();
+            } catch (InterruptedException ex) {
+                throw new InterruptedIOException();
+            } catch (ExecutionException ex) {
+                // transfer failed
+                Throwable e = ex.getCause();
+                if (e instanceof IOException) {
+                    throw (IOException) e;
+                } else if (e instanceof RuntimeException) {
+                    throw (RuntimeException) e;
+                } else if (e instanceof Error) {
+                    throw (Error) e;
+                } else {
+                    throw new IOException(e);
+                }
+            }
+        }
+    }
+
+    public class WriterChannel implements WritableByteChannel
+    {
+        public int write(ByteBuffer src) throws IOException
+        {
+            int sz = src.remaining();
+            if (sz <= 0) {
+                return sz;
+            }
+
+            synchronized(readerChannel) {
+                if (!readerChannel.waitForWritable()) {
+                    return -1;
+                }
+
+                readerChannel.setBuffer(src);
+
+                if (!readerChannel.waitForWritable()) {  // wait for complete processing src
+                    return -1;
+                }
+            }
+
+            return sz - src.remaining();
+        }
+
+        public boolean isOpen()
+        {
+            return readerChannel.isOpen();
+        }
+
+        public void close() throws IOException
+        {
+            readerChannel.closePeer();
+            waitForTransferCompletion();
+        }
+    }
+
+    private static int transferByteBuffer(ByteBuffer src, ByteBuffer dst)
+    {
+        int pos = dst.position();
+
+        int srcrem = src.remaining();
+        int dstrem = dst.remaining();
+        if (dstrem < srcrem) {
+            int lim = src.limit();
+            try {
+                src.limit(src.position() + dstrem);
+                dst.put(src);
+            } finally {
+                src.limit(lim);
+            }
+        } else {
+            dst.put(src);
+        }
+
+        return dst.position() - pos;
+    }
+
+    public class ReaderChannel implements ReadableByteChannel
+    {
+        private ByteBuffer buffer;
+        private Throwable exception;
+
+        public synchronized int read(ByteBuffer dst) throws IOException
+        {
+            if (!waitForReadable()) {
+                return -1;
+            }
+
+            int len = transferByteBuffer(buffer, dst);
+            if (!buffer.hasRemaining()) {
+                setBuffer(null);
+                notifyAll();
+            }
+
+            return len;
+        }
+
+        public synchronized boolean isOpen()
+        {
+            return exception == null;
+        }
+
+        public void close() throws IOException
+        {
+            setException(new EOFException("reader closed channel"));
+        }
+
+        private void setBuffer(ByteBuffer buffer)
+        {
+            this.buffer = buffer;
+            notifyAll();
+        }
+
+        private synchronized boolean waitForWritable() throws IOException
+        {
+            while (buffer != null) {
+                if (exception != null) {
+                    if (exception instanceof EOFException) {
+                        return false;
+                    }
+                    throwException();
+                }
+
+                try {
+                    wait();
+                } catch (InterruptedException ex) {
+                    // TODO throws ClosedByInterruptException or InterruptedIOException?
+                }
+            }
+
+            return true;
+        }
+
+        private boolean waitForReadable() throws IOException
+        {
+            while(buffer == null) {
+                if (exception != null) {
+                    if (exception instanceof EOFException) {
+                        return false;
+                    }
+                    throwException();
+                }
+
+                try {
+                    wait();
+                } catch (InterruptedException ex) {
+                    // TODO throws ClosedByInterruptException or InterruptedIOException?
+                }
+            }
+
+            return true;
+        }
+
+        public synchronized void closePeer() throws IOException
+        {
+            waitForWritable();
+            if( exception != null && !(exception instanceof EOFException)) {
+                throwException();
+            }
+            setException(new EOFException("writer closed channel"));
+        }
+
+        public synchronized void setException(Throwable exception)
+        {
+            if (this.exception == null) {
+                this.exception = exception;
+            }
+            notifyAll();
+        }
+
+        public synchronized void overwriteException(Throwable exception)
+        {
+            this.exception = exception;
+            notifyAll();
+        }
+
+        public boolean hasException()
+        {
+            return exception != null;
+        }
+
+        public void throwException() throws IOException
+        {
+            Throwable ex = exception;
+            if (ex instanceof IOException) {
+                throw (IOException) ex;
+            } else if (ex instanceof RuntimeException) {
+                throw (RuntimeException) ex;
+            } else if (ex instanceof Error) {
+                throw (Error) ex;
+            } else {
+                throw new IOException(ex);
+            }
+        }
+    }
+}
+

data/src/main/java/org/embulk/input/ftp/RetryExecutor.java

@@ -0,0 +1,131 @@
+// TODO copied from S3FileInputPlugin. This should be moved to org.embulk.
+package org.embulk.input.ftp;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+public class RetryExecutor
+{
+    public static RetryExecutor retryExecutor()
+    {
+        // TODO default configuration
+        return new RetryExecutor(3, 500, 30*60*1000);
+    }
+
+    public static class RetryGiveupException
+            extends ExecutionException
+    {
+        public RetryGiveupException(String message, Exception cause)
+        {
+            super(cause);
+        }
+
+        public RetryGiveupException(Exception cause)
+        {
+            super(cause);
+        }
+
+        public Exception getCause()
+        {
+            return (Exception) super.getCause();
+        }
+    }
+
+    public static interface Retryable<T>
+            extends Callable<T>
+    {
+        public T call()
+            throws Exception;
+
+        public boolean isRetryableException(Exception exception);
+
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            throws RetryGiveupException;
+
+        public void onGiveup(Exception firstException, Exception lastException)
+            throws RetryGiveupException;
+    }
+
+    private final int retryLimit;
+    private final int initialRetryWait;
+    private final int maxRetryWait;
+
+    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
+    {
+        this.retryLimit = retryLimit;
+        this.initialRetryWait = initialRetryWait;
+        this.maxRetryWait = maxRetryWait;
+    }
+
+    public RetryExecutor withRetryLimit(int count)
+    {
+        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
+    }
+
+    public RetryExecutor withInitialRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, msec, maxRetryWait);
+    }
+
+    public RetryExecutor withMaxRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, initialRetryWait, msec);
+    }
+
+    public <T> T runInterruptible(Retryable<T> op)
+            throws InterruptedException, RetryGiveupException
+    {
+        return run(op, true);
+    }
+
+    public <T> T run(Retryable<T> op)
+            throws RetryGiveupException
+    {
+        try {
+            return run(op, false);
+        } catch (InterruptedException ex) {
+            throw new RetryGiveupException("Unexpected interruption", ex);
+        }
+    }
+
+    private <T> T run(Retryable<T> op, boolean interruptible)
+            throws InterruptedException, RetryGiveupException
+    {
+        int retryWait = initialRetryWait;
+        int retryCount = 0;
+
+        Exception firstException = null;
+
+        while(true) {
+            try {
+                return op.call();
+            } catch (Exception exception) {
+                if (firstException == null) {
+                    firstException = exception;
+                }
+                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
+                    op.onGiveup(firstException, exception);
+                    throw new RetryGiveupException(firstException);
+                }
+
+                retryCount++;
+                op.onRetry(exception, retryCount, retryLimit, retryWait);
+
+                try {
+                    Thread.sleep(retryWait);
+                } catch (InterruptedException ex) {
+                    if (interruptible) {
+                        throw ex;
+                    }
+                }
+
+                // exponential back-off with hard limit
+                retryWait *= 2;
+                if (retryWait > maxRetryWait) {
+                    retryWait = maxRetryWait;
+                }
+            }
+        }
+    }
+}
+

data/src/main/java/org/embulk/input/ftp/RetryableInputStream.java

@@ -0,0 +1,129 @@
+// TODO copied from S3FileInputPlugin. This should be moved to org.embulk.
+package org.embulk.input.ftp;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+public class RetryableInputStream
+        extends InputStream
+{
+    public interface Opener
+    {
+        public InputStream open(long offset, Exception exception) throws IOException;
+    }
+
+    private final Opener opener;
+    protected InputStream in;
+    private long offset;
+    private long markedOffset;
+
+    public RetryableInputStream(InputStream initialInputStream, Opener reopener)
+    {
+        this.opener = reopener;
+        this.in = initialInputStream;
+        this.offset = 0L;
+        this.markedOffset = 0L;
+    }
+
+    public RetryableInputStream(Opener opener) throws IOException
+    {
+        this(opener.open(0, null), opener);
+    }
+
+    private void reopen(Exception exception) throws IOException
+    {
+        if (in != null) {
+            in.close();
+            in = null;
+        }
+        in = opener.open(offset, exception);
+    }
+
+    @Override
+    public int read() throws IOException
+    {
+        while (true) {
+            try {
+                int v = in.read();
+                offset += 1;
+                return v;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b, off, len);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public long skip(long n) throws IOException
+    {
+        while (true) {
+            try {
+                long r = in.skip(n);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int available() throws IOException
+    {
+        return in.available();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        in.close();
+    }
+
+    @Override
+    public void mark(int readlimit)
+    {
+        in.mark(readlimit);
+        markedOffset = offset;
+    }
+
+    @Override
+    public void reset() throws IOException
+    {
+        in.reset();
+        offset = markedOffset;
+    }
+
+    @Override
+    public boolean markSupported()
+    {
+        return in.markSupported();
+    }
+}