embulk 0.7.0-java

data/embulk-core/src/main/java/org/embulk/spi/util/Pages.java

@@ -0,0 +1,139 @@
+package org.embulk.spi.util;
+
+import java.util.List;
+import java.util.Iterator;
+import com.google.common.collect.ImmutableList;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.Column;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+
+public class Pages
+{
+    public static List<Object[]> toObjects(Schema schema, Page page)
+    {
+        return toObjects(schema, ImmutableList.of(page));
+    }
+
+    // TODO use streaming and return Iterable
+    public static List<Object[]> toObjects(Schema schema, Iterable<Page> pages)
+    {
+        ImmutableList.Builder<Object[]> builder = ImmutableList.builder();
+        Iterator<Page> ite = pages.iterator();
+        try (PageReader reader = new PageReader(schema)) {
+            while (ite.hasNext()) {
+                reader.setPage(ite.next());
+                while (reader.nextRecord()) {
+                    builder.add(toObjects(reader));
+                }
+            }
+        }
+        return builder.build();
+    }
+
+    public static Object[] toObjects(final PageReader record)
+    {
+        final Object[] values = new Object[record.getSchema().getColumns().size()];
+        record.getSchema().visitColumns(new ObjectColumnVisitor(record) {
+            @Override
+            public void visit(Column column, Object object)
+            {
+                values[column.getIndex()] = object;
+            }
+        });
+        return values;
+    }
+
+    public static abstract class ObjectColumnVisitor
+            implements ColumnVisitor
+    {
+        private final PageReader record;
+
+        public ObjectColumnVisitor(PageReader record)
+        {
+            this.record = record;
+        }
+
+        public abstract void visit(Column column, Object obj);
+
+        @Override
+        public void booleanColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getBoolean(column));
+            }
+        }
+
+        @Override
+        public void longColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getLong(column));
+            }
+        }
+
+        @Override
+        public void doubleColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getDouble(column));
+            }
+        }
+
+        @Override
+        public void stringColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getString(column));
+            }
+        }
+
+        @Override
+        public void timestampColumn(Column column)
+        {
+            if (record.isNull(column)) {
+                visit(column, null);
+            } else {
+                visit(column, record.getTimestamp(column));
+            }
+        }
+    }
+
+    public static Object getObject(PageReader record, Column column)
+    {
+        GetObjectColumnVisitor visitor = new GetObjectColumnVisitor(record);
+        column.visit(visitor);
+        return visitor.get();
+    }
+
+    private static class GetObjectColumnVisitor
+            extends ObjectColumnVisitor
+    {
+        private Object object;
+
+        public GetObjectColumnVisitor(PageReader record)
+        {
+            super(record);
+        }
+
+        public Object get()
+        {
+            return object;
+        }
+
+        public void visit(Column column, Object object)
+        {
+            this.object = object;
+        }
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java

@@ -0,0 +1,128 @@
+package org.embulk.spi.util;
+
+import java.io.InputStream;
+import java.io.IOException;
+
+public class ResumableInputStream
+        extends InputStream
+{
+    public interface Reopener
+    {
+        public InputStream reopen(long offset, Exception closedCause) throws IOException;
+    }
+
+    private final Reopener reopener;
+    protected InputStream in;
+    private long offset;
+    private long markedOffset;
+
+    public ResumableInputStream(InputStream initialInputStream, Reopener reopener)
+    {
+        this.reopener = reopener;
+        this.in = initialInputStream;
+        this.offset = 0L;
+        this.markedOffset = 0L;
+    }
+
+    public ResumableInputStream(Reopener reopener) throws IOException
+    {
+        this(reopener.reopen(0, null), reopener);
+    }
+
+    private void reopen(Exception closedCause) throws IOException
+    {
+        if (in != null) {
+            in.close();
+            in = null;
+        }
+        in = reopener.reopen(offset, closedCause);
+    }
+
+    @Override
+    public int read() throws IOException
+    {
+        while (true) {
+            try {
+                int v = in.read();
+                offset += 1;
+                return v;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int read(byte[] b, int off, int len) throws IOException
+    {
+        while (true) {
+            try {
+                int r = in.read(b, off, len);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public long skip(long n) throws IOException
+    {
+        while (true) {
+            try {
+                long r = in.skip(n);
+                offset += r;
+                return r;
+            } catch (IOException | RuntimeException ex) {
+                reopen(ex);
+            }
+        }
+    }
+
+    @Override
+    public int available() throws IOException
+    {
+        return in.available();
+    }
+
+    @Override
+    public void close() throws IOException
+    {
+        in.close();
+    }
+
+    @Override
+    public void mark(int readlimit)
+    {
+        in.mark(readlimit);
+        markedOffset = offset;
+    }
+
+    @Override
+    public void reset() throws IOException
+    {
+        in.reset();
+        offset = markedOffset;
+    }
+
+    @Override
+    public boolean markSupported()
+    {
+        return in.markSupported();
+    }
+}

data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java

@@ -0,0 +1,130 @@
+package org.embulk.spi.util;
+
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+
+public class RetryExecutor
+{
+    public static RetryExecutor retryExecutor()
+    {
+        // TODO default configuration
+        return new RetryExecutor(3, 500, 30*60*1000);
+    }
+
+    public static class RetryGiveupException
+            extends ExecutionException
+    {
+        public RetryGiveupException(String message, Exception cause)
+        {
+            super(cause);
+        }
+
+        public RetryGiveupException(Exception cause)
+        {
+            super(cause);
+        }
+
+        public Exception getCause()
+        {
+            return (Exception) super.getCause();
+        }
+    }
+
+    public static interface Retryable<T>
+            extends Callable<T>
+    {
+        public T call()
+            throws Exception;
+
+        public boolean isRetryableException(Exception exception);
+
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            throws RetryGiveupException;
+
+        public void onGiveup(Exception firstException, Exception lastException)
+            throws RetryGiveupException;
+    }
+
+    private final int retryLimit;
+    private final int initialRetryWait;
+    private final int maxRetryWait;
+
+    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
+    {
+        this.retryLimit = retryLimit;
+        this.initialRetryWait = initialRetryWait;
+        this.maxRetryWait = maxRetryWait;
+    }
+
+    public RetryExecutor withRetryLimit(int count)
+    {
+        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
+    }
+
+    public RetryExecutor withInitialRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, msec, maxRetryWait);
+    }
+
+    public RetryExecutor withMaxRetryWait(int msec)
+    {
+        return new RetryExecutor(retryLimit, initialRetryWait, msec);
+    }
+
+    public <T> T runInterruptible(Retryable<T> op)
+            throws InterruptedException, RetryGiveupException
+    {
+        return run(op, true);
+    }
+
+    public <T> T run(Retryable<T> op)
+            throws RetryGiveupException
+    {
+        try {
+            return run(op, false);
+        } catch (InterruptedException ex) {
+            throw new RetryGiveupException("Unexpected interruption", ex);
+        }
+    }
+
+    private <T> T run(Retryable<T> op, boolean interruptible)
+            throws InterruptedException, RetryGiveupException
+    {
+        int retryWait = initialRetryWait;
+        int retryCount = 0;
+
+        Exception firstException = null;
+
+        while(true) {
+            try {
+                return op.call();
+            } catch (Exception exception) {
+                if (firstException == null) {
+                    firstException = exception;
+                }
+                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
+                    op.onGiveup(firstException, exception);
+                    throw new RetryGiveupException(firstException);
+                }
+
+                retryCount++;
+                op.onRetry(exception, retryCount, retryLimit, retryWait);
+
+                try {
+                    Thread.sleep(retryWait);
+                } catch (InterruptedException ex) {
+                    if (interruptible) {
+                        throw ex;
+                    }
+                }
+
+                // exponential back-off with hard limit
+                retryWait *= 2;
+                if (retryWait > maxRetryWait) {
+                    retryWait = maxRetryWait;
+                }
+            }
+        }
+    }
+}
+

data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java

@@ -0,0 +1,53 @@
+package org.embulk.spi.util;
+
+import java.util.Map;
+import com.google.common.base.Optional;
+import org.embulk.config.Task;
+import org.embulk.spi.Column;
+import org.embulk.spi.Schema;
+import org.embulk.spi.ColumnConfig;
+import org.embulk.spi.SchemaConfig;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.type.TimestampType;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.SchemaConfig;
+
+public class Timestamps
+{
+    private Timestamps() { }
+
+    private interface TimestampColumnOption
+            extends Task, TimestampParser.TimestampColumnOption
+    { }
+
+    public static TimestampParser[] newTimestampColumnParsers(
+            TimestampParser.Task parserTask, SchemaConfig schema)
+    {
+        TimestampParser[] parsers = new TimestampParser[schema.getColumnCount()];
+        int i = 0;
+        for (ColumnConfig column : schema.getColumns()) {
+            if (column.getType() instanceof TimestampType) {
+                TimestampColumnOption option = column.getOption().loadConfig(TimestampColumnOption.class);
+                parsers[i] = new TimestampParser(parserTask, option);
+            }
+            i++;
+        }
+        return parsers;
+    }
+
+    public static TimestampFormatter[] newTimestampColumnFormatters(
+            TimestampFormatter.Task formatterTask, Schema schema,
+            Map<String, ? extends TimestampFormatter.TimestampColumnOption> columnOptions)
+    {
+        TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()];
+        int i = 0;
+        for (Column column : schema.getColumns()) {
+            if (column.getType() instanceof TimestampType) {
+                Optional<TimestampFormatter.TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName()));
+                formatters[i] = new TimestampFormatter(formatterTask, option);
+            }
+            i++;
+        }
+        return formatters;
+    }
+}