embulk 0.8.39-java → 0.10.24-java

data/embulk-core/src/main/java/org/embulk/spi/util/ResumableInputStream.java

@@ -1,168 +0,0 @@
-package org.embulk.spi.util;
-
-import java.io.InputStream;
-import java.io.IOException;
-
-public class ResumableInputStream
-        extends InputStream
-{
-    public interface Reopener
-    {
-        public InputStream reopen(long offset, Exception closedCause) throws IOException;
-    }
-
-    private final Reopener reopener;
-    protected InputStream in;
-    private long offset;
-    private long markedOffset;
-    private Exception lastClosedCause;
-    private boolean closed;
-
-    public ResumableInputStream(InputStream initialInputStream, Reopener reopener)
-    {
-        this.reopener = reopener;
-        this.in = initialInputStream;
-        this.offset = 0L;
-        this.markedOffset = 0L;
-        this.lastClosedCause = null;
-    }
-
-    public ResumableInputStream(Reopener reopener) throws IOException
-    {
-        this(reopener.reopen(0, null), reopener);
-    }
-
-    private void reopen(Exception closedCause) throws IOException
-    {
-        if (in != null) {
-            lastClosedCause = closedCause;
-            try {
-                in.close();
-            } catch (IOException ignored) {
-            }
-            in = null;
-        }
-        in = reopener.reopen(offset, closedCause);
-        lastClosedCause = null;
-    }
-
-    @Override
-    public int read() throws IOException
-    {
-        ensureOpened();
-        while (true) {
-            try {
-                int v = in.read();
-                offset += 1;
-                return v;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int read(byte[] b) throws IOException
-    {
-        ensureOpened();
-        while (true) {
-            try {
-                int r = in.read(b);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int read(byte[] b, int off, int len) throws IOException
-    {
-        ensureOpened();
-        while (true) {
-            try {
-                int r = in.read(b, off, len);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public long skip(long n) throws IOException
-    {
-        ensureOpened();
-        while (true) {
-            try {
-                long r = in.skip(n);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int available() throws IOException
-    {
-        ensureOpened();
-        return in.available();
-    }
-
-    @Override
-    public void close() throws IOException
-    {
-        if (in != null) {
-            in.close();
-            closed = true;
-            in = null;
-        }
-    }
-
-    @Override
-    public void mark(int readlimit)
-    {
-        try {
-            ensureOpened();
-        }
-        catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-        in.mark(readlimit);
-        markedOffset = offset;
-    }
-
-    @Override
-    public void reset() throws IOException
-    {
-        ensureOpened();
-        in.reset();
-        offset = markedOffset;
-    }
-
-    @Override
-    public boolean markSupported()
-    {
-        try {
-            ensureOpened();
-        }
-        catch (IOException ex) {
-            throw new RuntimeException(ex);
-        }
-        return in.markSupported();
-    }
-
-    private void ensureOpened() throws IOException
-    {
-        if (in == null) {
-            if (closed) {
-                throw new IOException("stream closed");
-            }
-            reopen(lastClosedCause);
-        }
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/RetryExecutor.java

@@ -1,130 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-public class RetryExecutor
-{
-    public static RetryExecutor retryExecutor()
-    {
-        // TODO default configuration
-        return new RetryExecutor(3, 500, 30*60*1000);
-    }
-
-    public static class RetryGiveupException
-            extends ExecutionException
-    {
-        public RetryGiveupException(String message, Exception cause)
-        {
-            super(cause);
-        }
-
-        public RetryGiveupException(Exception cause)
-        {
-            super(cause);
-        }
-
-        public Exception getCause()
-        {
-            return (Exception) super.getCause();
-        }
-    }
-
-    public static interface Retryable<T>
-            extends Callable<T>
-    {
-        public T call()
-            throws Exception;
-
-        public boolean isRetryableException(Exception exception);
-
-        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-            throws RetryGiveupException;
-
-        public void onGiveup(Exception firstException, Exception lastException)
-            throws RetryGiveupException;
-    }
-
-    private final int retryLimit;
-    private final int initialRetryWait;
-    private final int maxRetryWait;
-
-    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
-    {
-        this.retryLimit = retryLimit;
-        this.initialRetryWait = initialRetryWait;
-        this.maxRetryWait = maxRetryWait;
-    }
-
-    public RetryExecutor withRetryLimit(int count)
-    {
-        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
-    }
-
-    public RetryExecutor withInitialRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, msec, maxRetryWait);
-    }
-
-    public RetryExecutor withMaxRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, initialRetryWait, msec);
-    }
-
-    public <T> T runInterruptible(Retryable<T> op)
-            throws InterruptedException, RetryGiveupException
-    {
-        return run(op, true);
-    }
-
-    public <T> T run(Retryable<T> op)
-            throws RetryGiveupException
-    {
-        try {
-            return run(op, false);
-        } catch (InterruptedException ex) {
-            throw new RetryGiveupException("Unexpected interruption", ex);
-        }
-    }
-
-    private <T> T run(Retryable<T> op, boolean interruptible)
-            throws InterruptedException, RetryGiveupException
-    {
-        int retryWait = initialRetryWait;
-        int retryCount = 0;
-
-        Exception firstException = null;
-
-        while(true) {
-            try {
-                return op.call();
-            } catch (Exception exception) {
-                if (firstException == null) {
-                    firstException = exception;
-                }
-                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
-                    op.onGiveup(firstException, exception);
-                    throw new RetryGiveupException(firstException);
-                }
-
-                retryCount++;
-                op.onRetry(exception, retryCount, retryLimit, retryWait);
-
-                try {
-                    Thread.sleep(retryWait);
-                } catch (InterruptedException ex) {
-                    if (interruptible) {
-                        throw ex;
-                    }
-                }
-
-                // exponential back-off with hard limit
-                retryWait *= 2;
-                if (retryWait > maxRetryWait) {
-                    retryWait = maxRetryWait;
-                }
-            }
-        }
-    }
-}
-

data/embulk-core/src/main/java/org/embulk/spi/util/Timestamps.java

@@ -1,53 +0,0 @@
-package org.embulk.spi.util;
-
-import java.util.Map;
-import com.google.common.base.Optional;
-import org.embulk.config.Task;
-import org.embulk.spi.Column;
-import org.embulk.spi.Schema;
-import org.embulk.spi.ColumnConfig;
-import org.embulk.spi.SchemaConfig;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.spi.type.TimestampType;
-import org.embulk.spi.time.TimestampParser;
-import org.embulk.spi.SchemaConfig;
-
-public class Timestamps
-{
-    private Timestamps() { }
-
-    private interface TimestampColumnOption
-            extends Task, TimestampParser.TimestampColumnOption
-    { }
-
-    public static TimestampParser[] newTimestampColumnParsers(
-            TimestampParser.Task parserTask, SchemaConfig schema)
-    {
-        TimestampParser[] parsers = new TimestampParser[schema.getColumnCount()];
-        int i = 0;
-        for (ColumnConfig column : schema.getColumns()) {
-            if (column.getType() instanceof TimestampType) {
-                TimestampColumnOption option = column.getOption().loadConfig(TimestampColumnOption.class);
-                parsers[i] = new TimestampParser(parserTask, option);
-            }
-            i++;
-        }
-        return parsers;
-    }
-
-    public static TimestampFormatter[] newTimestampColumnFormatters(
-            TimestampFormatter.Task formatterTask, Schema schema,
-            Map<String, ? extends TimestampFormatter.TimestampColumnOption> columnOptions)
-    {
-        TimestampFormatter[] formatters = new TimestampFormatter[schema.getColumnCount()];
-        int i = 0;
-        for (Column column : schema.getColumns()) {
-            if (column.getType() instanceof TimestampType) {
-                Optional<TimestampFormatter.TimestampColumnOption> option = Optional.fromNullable(columnOptions.get(column.getName()));
-                formatters[i] = new TimestampFormatter(formatterTask, option);
-            }
-            i++;
-        }
-        return formatters;
-    }
-}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/AbstractDynamicColumnSetter.java

@@ -1,94 +0,0 @@
-package org.embulk.spi.util.dynamic;
-
-import org.jruby.runtime.builtin.IRubyObject;
-import org.jruby.RubyNil;
-import org.jruby.RubyBoolean;
-import org.jruby.RubyInteger;
-import org.jruby.RubyFloat;
-import org.jruby.RubyString;
-import org.jruby.RubyTime;
-import org.jruby.exceptions.RaiseException;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.Column;
-import org.embulk.spi.util.DynamicColumnSetter;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.json.RubyValueApi;
-import org.msgpack.value.Value;
-
-public abstract class AbstractDynamicColumnSetter
-        implements DynamicColumnSetter
-{
-    protected final PageBuilder pageBuilder;
-    protected final Column column;
-    protected final DefaultValueSetter defaultValue;
-
-    protected AbstractDynamicColumnSetter(PageBuilder pageBuilder, Column column,
-            DefaultValueSetter defaultValue)
-    {
-        this.pageBuilder = pageBuilder;
-        this.column = column;
-        this.defaultValue = defaultValue;
-    }
-
-    public abstract void setNull();
-
-    public abstract void set(boolean value);
-
-    public abstract void set(long value);
-
-    public abstract void set(double value);
-
-    public abstract void set(String value);
-
-    public abstract void set(Timestamp value);
-
-    public abstract void set(Value value);
-
-    @Deprecated
-    public void setRubyObject(IRubyObject rubyObject)
-    {
-        if (!deprecationWarned) {
-            System.err.println("[WARN] Plugin uses deprecated org.embulk.spi.util.dynamic.AbstractDynamicColumnSetter#setRubyObject");
-            System.err.println("[WARN] Report plugins in your config at: https://github.com/embulk/embulk/issues/799");
-            // The |deprecationWarned| flag is used only for warning messages.
-            // Even in case of race conditions, messages are just duplicated -- should be acceptable.
-            deprecationWarned = true;
-        }
-
-        if (rubyObject == null || rubyObject instanceof RubyNil) {
-            setNull();
-        } else if (rubyObject instanceof RubyBoolean) {
-            RubyBoolean b = (RubyBoolean) rubyObject;
-            set(b.isTrue());
-        } else if (rubyObject instanceof RubyInteger) {
-            RubyInteger i = (RubyInteger) rubyObject;
-            try {
-                set(i.getLongValue());
-            } catch (RaiseException ex) {
-                if ("RangeError".equals(ex.getException().getMetaClass().getBaseName())) {
-                    // integer is too large
-                    throw ex;  //TODO setDefaultValue();
-                } else {
-                    throw ex;
-                }
-            }
-        } else if (rubyObject instanceof RubyFloat) {
-            RubyFloat f = (RubyFloat) rubyObject;
-            set(f.getDoubleValue());
-        } else if (rubyObject instanceof RubyString) {
-            RubyString s = (RubyString) rubyObject;
-            set(s.asJavaString());
-        } else if (rubyObject instanceof RubyTime) {
-            RubyTime time = (RubyTime) rubyObject;
-            long msec = time.getDateTime().getMillis();
-            long nsec = time.getNSec();
-            long sec = msec / 1000 + nsec / 1000000000;
-            int nano = (int) ((msec % 1000) * 1000000 + nsec % 1000000000);
-            set(Timestamp.ofEpochSecond(sec, nano));
-        } else {
-            set(RubyValueApi.toValue(rubyObject.getRuntime(), rubyObject));
-        }
-    }
-
-    private static boolean deprecationWarned = false;
-}

data/embulk-core/src/main/java/org/embulk/spi/util/dynamic/BooleanColumnSetter.java

@@ -1,71 +0,0 @@
-package org.embulk.spi.util.dynamic;
-
-import com.google.common.collect.ImmutableSet;
-import org.embulk.spi.Column;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.time.Timestamp;
-import org.msgpack.value.Value;
-
-public class BooleanColumnSetter
-        extends AbstractDynamicColumnSetter
-{
-    private static final ImmutableSet<String> TRUE_STRINGS =
-        ImmutableSet.of(
-                "true", "True", "TRUE",
-                "yes", "Yes", "YES",
-                "t", "T", "y", "Y",
-                "on", "On", "ON",
-                "1");
-
-    public BooleanColumnSetter(PageBuilder pageBuilder, Column column,
-            DefaultValueSetter defaultValue)
-    {
-        super(pageBuilder, column, defaultValue);
-    }
-
-    @Override
-    public void setNull()
-    {
-        pageBuilder.setNull(column);
-    }
-
-    @Override
-    public void set(boolean v)
-    {
-        pageBuilder.setBoolean(column, v);
-    }
-
-    @Override
-    public void set(long v)
-    {
-        pageBuilder.setBoolean(column, v > 0);
-    }
-
-    @Override
-    public void set(double v)
-    {
-        pageBuilder.setBoolean(column, v > 0.0);
-    }
-
-    @Override
-    public void set(String v)
-    {
-        if (TRUE_STRINGS.contains(v)) {
-            pageBuilder.setBoolean(column, true);
-        } else {
-            defaultValue.setBoolean(pageBuilder, column);
-        }
-    }
-
-    @Override
-    public void set(Timestamp v)
-    {
-        defaultValue.setBoolean(pageBuilder, column);
-    }
-
-    @Override
-    public void set(Value v)
-    {
-        defaultValue.setBoolean(pageBuilder, column);
-    }
-}