embulk-input-s3 0.2.16 → 0.2.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95a2e3a70de76cdd69c0b0252f37b32671ce3896
-  data.tar.gz: 0f7a8a710a28478d528df8a31d08b996a0595772
+  metadata.gz: 3c9e972f5326281de40fc46190dc3d268282b34e
+  data.tar.gz: 0c63221f881710e6c4a18dbf10ab73be701193ff
 SHA512:
-  metadata.gz: 6ae53adc0606da4ec1e2a72a742b18b27ea40145b27baf5375e905670f243b411d43e15a55fdeb2172cc158e1e83b257f290a79d2ff92aa890f9ab57e38d2174
-  data.tar.gz: 332d2963cfde7edf02ed80b9fda7d651a12c490e03f7f2984cee329858b978ebd5c2d5401e9ea30efc5e4a55e66ccd6d87d285a908bd3a4819dffa50de52591e
+  metadata.gz: 3983830d6e700a3b3e0a2cdadd063613c4ff41b39e01c96c00c2f8a33515fd8d8be601c644184119bb87e46d1eab394869e91cf1c557c6420ef022f40a93b31e
+  data.tar.gz: 27471ec5bbc698865ab44d2d4ca94b2074c7589c5c9c0ce919b3c3b5bc18b885b6566b48fb74803c99abc10c04716771a1b9faa231ff780de3740da55b468358

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -6,15 +6,17 @@ import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -30,15 +32,13 @@ import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.ResumableInputStream;
-import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
-import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.embulk.spi.util.RetryExecutor;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
 import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
-import java.io.InterruptedIOException;
 import java.util.Iterator;
 import java.util.List;
 
@@ -50,13 +50,18 @@ public abstract class AbstractS3FileInputPlugin
     private static final Logger LOGGER = Exec.getLogger(S3FileInputPlugin.class);
 
     public interface PluginTask
-            extends AwsCredentialsTask, FileList.Task, Task
+            extends AwsCredentialsTask, FileList.Task, RetrySupportPluginTask, Task
     {
         @Config("bucket")
         public String getBucket();
 
         @Config("path_prefix")
-        public String getPathPrefix();
+        @ConfigDefault("null")
+        public Optional<String> getPathPrefix();
+
+        @Config("path")
+        @ConfigDefault("null")
+        public Optional<String> getPath();
 
         @Config("last_path")
         @ConfigDefault("null")
@@ -97,6 +102,7 @@ public abstract class AbstractS3FileInputPlugin
     {
         PluginTask task = config.loadConfig(getTaskClass());
 
+        validateInputTask(task);
         // list files recursively
         task.setFiles(listFiles(task));
 
@@ -208,21 +214,42 @@ public abstract class AbstractS3FileInputPlugin
         }
     }
 
-    private FileList listFiles(PluginTask task)
+    /**
+     * Build the common retry executor from some configuration params of plugin task.
+     * @param task Plugin task.
+     * @return RetryExecutor object
+     */
+    private static RetryExecutor retryExecutorFrom(RetrySupportPluginTask task)
+    {
+        return retryExecutor()
+            .withRetryLimit(task.getMaximumRetries())
+            .withInitialRetryWait(task.getInitialRetryIntervalMillis())
+            .withMaxRetryWait(task.getMaximumRetryIntervalMillis());
+    }
+
+    private FileList listFiles(final PluginTask task)
     {
-        LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix());
         try {
             AmazonS3 client = newS3Client(task);
             String bucketName = task.getBucket();
+            FileList.Builder builder = new FileList.Builder(task);
+            RetryExecutor retryExec = retryExecutorFrom(task);
+            if (task.getPath().isPresent()) {
+                LOGGER.info("Start getting object with path: [{}]", task.getPath().get());
+                addS3DirectObject(builder, client, task.getBucket(), task.getPath().get(), retryExec);
+            }
+            else {
+                // does not need to verify existent path prefix here since there is the validation requires either path or path_prefix
+                LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix().get());
+                if (task.getPathPrefix().get().equals("/")) {
+                    LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
+                }
 
-            if (task.getPathPrefix().equals("/")) {
-                LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
+                listS3FilesByPrefix(builder, client, bucketName,
+                        task.getPathPrefix().get(), task.getLastPath(), task.getSkipGlacierObjects(), retryExec);
+                LOGGER.info("Found total [{}] files", builder.size());
             }
 
-            FileList.Builder builder = new FileList.Builder(task);
-            listS3FilesByPrefix(builder, client, bucketName,
-                    task.getPathPrefix(), task.getLastPath(), task.getSkipGlacierObjects());
-            LOGGER.info("Found total [{}] files", builder.size());
             return builder.build();
         }
         catch (AmazonServiceException ex) {
@@ -236,11 +263,55 @@ public abstract class AbstractS3FileInputPlugin
             }
             throw ex;
         }
-        catch (InterruptedException | RetryGiveupException ex) {
-            throw new RuntimeException(ex);
+    }
+
+    @VisibleForTesting
+    public void addS3DirectObject(FileList.Builder builder,
+                                  final AmazonS3 client,
+                                  String bucket,
+                                  String objectKey)
+    {
+        addS3DirectObject(builder, client, bucket, objectKey, null);
+    }
+
+    @VisibleForTesting
+    public void addS3DirectObject(FileList.Builder builder,
+                                   final AmazonS3 client,
+                                   String bucket,
+                                   String objectKey,
+                                   RetryExecutor retryExec)
+    {
+        final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey);
+
+        ObjectMetadata objectMetadata = new AlwaysRetryable<ObjectMetadata>("Looking up for a single object") {
+            @Override
+            public ObjectMetadata call()
+            {
+                return client.getObjectMetadata(objectMetadataRequest);
+            }
+        }.executeWith(retryExec);
+
+        builder.add(objectKey, objectMetadata.getContentLength());
+    }
+
+    private void validateInputTask(PluginTask task)
+    {
+        if (!task.getPathPrefix().isPresent() && !task.getPath().isPresent()) {
+            throw new ConfigException("Either path or path_prefix is required");
         }
     }
 
+    @VisibleForTesting
+    public static void listS3FilesByPrefix(FileList.Builder builder,
+                                           final AmazonS3 client,
+                                           String bucketName,
+                                           String prefix,
+                                           Optional<String> lastPath,
+                                           boolean skipGlacierObjects)
+    {
+        listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null);
+    }
+
     /**
      * Lists S3 filenames filtered by prefix.
      * <p>
@@ -251,30 +322,28 @@ public abstract class AbstractS3FileInputPlugin
      * @param prefix Amazon S3 bucket name prefix
      * @param lastPath last path
      * @param skipGlacierObjects skip gracier objects
-     * @throws RetryGiveupException error when retrying
-     * @throws InterruptedException error when retrying
+     * @param retryExec a retry executor object to do the retrying
      */
+    @VisibleForTesting
     public static void listS3FilesByPrefix(FileList.Builder builder,
-                                           final AmazonS3 client, final String bucketName,
-                                           final String prefix, Optional<String> lastPath, boolean skipGlacierObjects) throws RetryGiveupException, InterruptedException
+                                           final AmazonS3 client,
+                                           String bucketName,
+                                           String prefix,
+                                           Optional<String> lastPath,
+                                           boolean skipGlacierObjects,
+                                           RetryExecutor retryExec)
     {
         String lastKey = lastPath.orNull();
         do {
             final String finalLastKey = lastKey;
-            Optional<ObjectListing> optOl = S3FileInputUtils.executeWithRetry(3, 500, 30 * 1000, new S3FileInputUtils.AlwaysRetryRetryable<Optional<ObjectListing>>()
-            {
+            final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
+            ObjectListing ol = new AlwaysRetryable<ObjectListing>("Listing objects") {
                 @Override
-                public Optional<ObjectListing> call() throws AmazonServiceException
+                public ObjectListing call()
                 {
-                    ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
-                    ObjectListing ol = client.listObjects(req);
-                    return Optional.of(ol);
+                    return client.listObjects(req);
                 }
-            });
-            if (!optOl.isPresent()) {
-                break;
-            }
-            ObjectListing ol = optOl.get();
+            }.executeWith(retryExec);
             for (S3ObjectSummary s : ol.getObjectSummaries()) {
                 if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
                     if (skipGlacierObjects) {
@@ -313,67 +382,34 @@ public abstract class AbstractS3FileInputPlugin
         private final AmazonS3 client;
         private final GetObjectRequest request;
         private final long contentLength;
+        private final RetryExecutor retryExec;
 
         public S3InputStreamReopener(AmazonS3 client, GetObjectRequest request, long contentLength)
+        {
+            this(client, request, contentLength, null);
+        }
+
+        public S3InputStreamReopener(AmazonS3 client, GetObjectRequest request, long contentLength, RetryExecutor retryExec)
         {
             this.client = client;
             this.request = request;
             this.contentLength = contentLength;
+            this.retryExec = retryExec;
         }
 
         @Override
         public InputStream reopen(final long offset, final Exception closedCause) throws IOException
         {
-            try {
-                return retryExecutor()
-                        .withRetryLimit(3)
-                        .withInitialRetryWait(500)
-                        .withMaxRetryWait(30 * 1000)
-                        .runInterruptible(new Retryable<InputStream>()
-                        {
-                            @Override
-                            public InputStream call() throws InterruptedIOException
-                            {
-                                log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
-                                request.setRange(offset, contentLength - 1);  // [first, last]
-                                return client.getObject(request).getObjectContent();
-                            }
-
-                            @Override
-                            public boolean isRetryableException(Exception exception)
-                            {
-                                return true;  // TODO
-                            }
-
-                            @Override
-                            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                                    throws RetryGiveupException
-                            {
-                                String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                        retryCount, retryLimit, retryWait / 1000, exception.getMessage());
-                                if (retryCount % 3 == 0) {
-                                    log.warn(message, exception);
-                                }
-                                else {
-                                    log.warn(message);
-                                }
-                            }
-
-                            @Override
-                            public void onGiveup(Exception firstException, Exception lastException)
-                                    throws RetryGiveupException
-                            {
-                                log.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
-                            }
-                        });
-            }
-            catch (RetryGiveupException ex) {
-                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
-                throw Throwables.propagate(ex.getCause());
-            }
-            catch (InterruptedException ex) {
-                throw new InterruptedIOException();
-            }
+            log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+            request.setRange(offset, contentLength - 1);  // [first, last]
+
+            return new AlwaysRetryable<S3ObjectInputStream>("Opening the file") {
+                @Override
+                public S3ObjectInputStream call()
+                {
+                    return client.getObject(request).getObjectContent();
+                }
+            }.executeWithCheckedException(retryExec, IOException.class);
         }
     }
 
@@ -408,12 +444,14 @@ public abstract class AbstractS3FileInputPlugin
         private AmazonS3 client;
         private final String bucket;
         private final Iterator<String> iterator;
+        private final RetryExecutor retryExec;
 
         public SingleFileProvider(PluginTask task, int taskIndex)
         {
             this.client = newS3Client(task);
             this.bucket = task.getBucket();
             this.iterator = task.getFiles().get(taskIndex).iterator();
+            this.retryExec = retryExecutorFrom(task);
         }
 
         @Override
@@ -427,7 +465,7 @@ public abstract class AbstractS3FileInputPlugin
             S3Object obj = client.getObject(request);
             long objectSize = obj.getObjectMetadata().getContentLength();
             LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
-            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, objectSize));
+            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec));
         }
 
         @Override

data/src/main/java/org/embulk/input/s3/AlwaysRetryable.java

@@ -0,0 +1,158 @@
+package org.embulk.input.s3;
+
+import com.google.common.base.Throwables;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.util.concurrent.Callable;
+
+import static java.lang.String.format;
+import static org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import static org.embulk.spi.util.RetryExecutor.Retryable;
+
+/**
+ * Always retry, regardless the occurred exceptions,
+ * Also provide a default approach for exception propagation.
+ */
+class AlwaysRetryable<T> implements Retryable<T>
+{
+    private static final Logger log = Exec.getLogger(AlwaysRetryable.class);
+
+    private String operationName;
+    private Callable<T> callable;
+
+    /**
+     * @param operationName the name that will be referred on logging
+     */
+    public AlwaysRetryable(String operationName)
+    {
+        this.operationName = operationName;
+    }
+
+    /**
+     * @param operationName the name that will be referred on logging
+     * @param callable the operation, either define this at construction time or override the call() method
+     */
+    public AlwaysRetryable(String operationName, Callable<T> callable)
+    {
+        this.operationName = operationName;
+        this.callable = callable;
+    }
+
+    public AlwaysRetryable()
+    {
+        this("Anonymous operation");
+    }
+
+    public AlwaysRetryable(Callable<T> callable)
+    {
+        this("Anonymous operation", callable);
+    }
+
+    @Override
+    public T call() throws Exception
+    {
+        if (callable != null) {
+            return callable.call();
+        }
+        else {
+            throw new IllegalStateException("Either override call() or construct with a Runnable");
+        }
+    }
+
+    @Override
+    public boolean isRetryableException(Exception exception)
+    {
+        return true;
+    }
+
+    @Override
+    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+    {
+        String message = format("%s failed. Retrying %d/%d after %d seconds. Message: %s",
+                operationName, retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+        if (retryCount % retryLimit == 0) {
+            log.warn(message, exception);
+        }
+        else {
+            log.warn(message);
+        }
+    }
+
+    @Override
+    public void onGiveup(Exception firstException, Exception lastException)
+    {
+        // Exceptions would be propagated, so it's up to the caller to handle, this is just warning
+        log.warn("Giving up on retrying for {}, first exception is [{}], last exception is [{}]",
+                operationName, firstException.getMessage(), lastException.getMessage());
+    }
+
+    /**
+     * Run itself by the supplied executor,
+     *
+     * This propagates all exceptions (as unchecked) and unwrap RetryGiveupException for the original cause.
+     * If the original exception already is a RuntimeException, it will be propagated as is. If not, it will
+     * be wrapped around with a RuntimeException.
+     *
+     * For convenient, it execute normally without retrying when executor is null.
+     *
+     * @throws RuntimeException the original cause
+     */
+    public T executeWith(RetryExecutor executor)
+    {
+        if (executor == null) {
+            try {
+                return this.call();
+            }
+            catch (Exception e) {
+                Throwables.propagate(e);
+            }
+        }
+
+        try {
+            return executor.runInterruptible(this);
+        }
+        catch (RetryGiveupException e) {
+            throw Throwables.propagate(e.getCause());
+        }
+        catch (InterruptedException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+
+    /**
+     * Run itself by the supplied executor,
+     *
+     * Same as `executeWith`, this propagates all original exceptions. But `propagateAsIsException` will
+     * be re-throw without being wrapped on a RuntimeException, whether it is a checked or unchecked exception.
+     *
+     * For convenient, it execute normally without retrying when executor is null.
+     *
+     * @throws X whatever checked exception that you decided to propagate directly
+     * @throws RuntimeException wrap around whatever the original cause of failure (potentially thread interruption)
+     */
+    public <X extends Throwable> T executeWithCheckedException(RetryExecutor executor,
+                                                               Class<X> propagateAsIsException) throws X
+    {
+        if (executor == null) {
+            try {
+                return this.call();
+            }
+            catch (Exception e) {
+                Throwables.propagate(e);
+            }
+        }
+
+        try {
+            return executor.runInterruptible(this);
+        }
+        catch (RetryGiveupException e) {
+            Throwables.propagateIfInstanceOf(e.getCause(), propagateAsIsException);
+            throw Throwables.propagate(e.getCause());
+        }
+        catch (InterruptedException e) {
+            throw Throwables.propagate(e);
+        }
+    }
+}

data/src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java

@@ -0,0 +1,20 @@
+package org.embulk.input.s3;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+
+public interface RetrySupportPluginTask extends Task
+{
+    @Config("maximum_retries")
+    @ConfigDefault("7")
+    int getMaximumRetries();
+
+    @Config("initial_retry_interval_millis")
+    @ConfigDefault("30000")
+    int getInitialRetryIntervalMillis();
+
+    @Config("maximum_retry_interval_millis")
+    @ConfigDefault("480000")
+    int getMaximumRetryIntervalMillis();
+}

data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java

@@ -0,0 +1,115 @@
+package org.embulk.input.s3;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.google.common.base.Optional;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.util.RetryExecutor;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.mock;
+
+public class TestAbstractS3FileInputPlugin
+{
+    private static RetryExecutor retryExecutor()
+    {
+        return RetryExecutor.retryExecutor()
+                .withInitialRetryWait(0)
+                .withMaxRetryWait(0);
+    }
+
+    private static AbstractS3FileInputPlugin dummyS3Plugin()
+    {
+        return new AbstractS3FileInputPlugin()
+        {
+            @Override
+            protected Class<? extends PluginTask> getTaskClass()
+            {
+                return PluginTask.class;
+            }
+        };
+    }
+
+    private static class SomeException extends RuntimeException
+    {
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private AmazonS3 client;
+
+    @Before
+    public void createResources()
+    {
+        client = mock(AmazonS3.class);
+    }
+
+    @Test
+    public void listS3FilesByPrefix()
+    {
+        doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true);
+    }
+
+    @Test
+    public void listS3FileByPrefix_with_retry()
+    {
+        doThrow(new RuntimeException()).doReturn(new ObjectListing())
+                .when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(
+                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
+                retryExecutor().withRetryLimit(1));
+    }
+
+    @Test(expected = SomeException.class)
+    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception()
+    {
+        doThrow(new SomeException()).doReturn(new ObjectListing())
+                .when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(
+                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
+                retryExecutor().withRetryLimit(0));
+    }
+
+    @Test
+    public void addS3DirectObject()
+    {
+        doReturn(new ObjectMetadata()).when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
+        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
+        dummyS3Plugin().addS3DirectObject(builder, client, "some_bucket", "some_prefix");
+    }
+
+    @Test
+    public void addS3DirectObject_with_retry()
+    {
+        doThrow(new RuntimeException()).doReturn(new ObjectMetadata())
+                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
+        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
+        dummyS3Plugin().addS3DirectObject(
+                builder, client, "some_bucket", "some_prefix",
+                retryExecutor());
+    }
+
+    @Test(expected = SomeException.class)
+    public void addS3DirectObject_on_retry_gave_up_should_throw_original_exception()
+    {
+        doThrow(new SomeException()).doReturn(new ObjectMetadata())
+                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
+        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
+        dummyS3Plugin().addS3DirectObject(
+                builder, client, "some_bucket", "some_prefix",
+                retryExecutor().withRetryLimit(0));
+    }
+}

data/src/test/java/org/embulk/input/s3/TestAlwaysRetryable.java

@@ -0,0 +1,131 @@
+package org.embulk.input.s3;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.spi.util.RetryExecutor;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.concurrent.Callable;
+
+import static java.lang.String.format;
+import static org.msgpack.core.Preconditions.checkArgument;
+
+public class TestAlwaysRetryable
+{
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();  // require for AlwaysRetryable's logger
+
+    private static class Deny extends RuntimeException implements Callable
+    {
+        private int pastCalls = 0;
+        private final int targetCalls;
+        private Exception exception;
+
+        Deny(int targetCalls)
+        {
+            super(format("Try harder! (Will pass after %d calls)", targetCalls));
+            checkArgument(targetCalls >= 0);
+            this.targetCalls = targetCalls;
+        }
+
+        static Deny until(int calls)
+        {
+            return new Deny(calls);
+        }
+
+        Deny with(Exception exception)
+        {
+            this.exception = exception;
+            return this;
+        }
+
+        @Override
+        public Object call() throws Exception
+        {
+            if (pastCalls < targetCalls) {
+                pastCalls++;
+                if (exception != null) {
+                    throw exception;
+                }
+                else {
+                    throw this;
+                }
+            }
+            pastCalls++;
+            return null;
+        }
+    }
+
+    private static RetryExecutor retryExecutor()
+    {
+        return RetryExecutor.retryExecutor()
+                .withInitialRetryWait(0)
+                .withMaxRetryWait(0);
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void guarantee_retry_attempts_just_like_Retryable() throws Exception
+    {
+        retryExecutor()
+                .withRetryLimit(0)
+                .run(new AlwaysRetryable(Deny.until(0)));
+        retryExecutor()
+                .withRetryLimit(1)
+                .run(new AlwaysRetryable(Deny.until(1)));
+        retryExecutor()
+                .withRetryLimit(2)
+                .run(new AlwaysRetryable(Deny.until(1)));
+        retryExecutor()
+                .withRetryLimit(3)
+                .run(new AlwaysRetryable(Deny.until(2)));
+    }
+
+    @Test(expected = RetryGiveupException.class)
+    @SuppressWarnings("unchecked")
+    public void fail_after_exceeding_attempts_just_like_Retryable() throws Exception
+    {
+        retryExecutor()
+                .withRetryLimit(3)
+                .run(new AlwaysRetryable(Deny.until(4)));
+    }
+
+    @Test(expected = Deny.class)
+    @SuppressWarnings("unchecked")
+    public void execute_should_unwrap_RetryGiveupException() throws Exception
+    {
+        new AlwaysRetryable(Deny.until(4))
+                .executeWith(retryExecutor().withRetryLimit(3));
+    }
+
+    @Test(expected = RuntimeException.class)
+    @SuppressWarnings("unchecked")
+    public void execute_should_unwrap_RetryGiveupException_but_rewrap_checked_exception_in_a_RuntimeException()
+    {
+        new AlwaysRetryable(Deny.until(4).with(new Exception("A checked exception")))
+                .executeWith(retryExecutor().withRetryLimit(3));
+    }
+
+    @Test(expected = IOException.class)
+    public void executeAndPropagateAsIs_should_leave_original_exception_unwrapped() throws IOException
+    {
+        RetryExecutor retryExc = retryExecutor().withRetryLimit(3);
+        // An explicit type parameter for operation return type is needed here,
+        // Without one, javac (at least on 1.8) will fails to infer the X exception type parameter.
+        new AlwaysRetryable<Object>() {
+            @Override
+            public Object call() throws IOException
+            {
+                throw new IOException();
+            }
+        }.executeWithCheckedException(retryExc, IOException.class);
+    }
+
+    @Test(expected = IllegalStateException.class)
+    public void execute_without_an_implementation_should_throw_an_IllegalStateException()
+    {
+        new AlwaysRetryable().executeWith(retryExecutor());
+    }
+}

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -161,6 +161,28 @@ public class TestS3FileInputPlugin
         }
     }
 
+    @Test
+    public void usePath()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX))
+                .set("path_prefix", null);
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+        assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path"));
+        assertRecords(config, output);
+    }
+
+    @Test
+    public void usePathAsHighPriorityThanPathPrefix()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("path", String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX))
+                .set("path_prefix", "foo"); // path_prefix has the bad value, if path_prefix is chosen, expected result will be failed
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+        assertEquals(String.format("%s/sample_01.csv", EMBULK_S3_TEST_PATH_PREFIX), configDiff.get(String.class, "last_path"));
+        assertRecords(config, output);
+    }
+
     @Test
     public void configuredEndpoint()
     {

data/src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java

@@ -1,5 +1,6 @@
 package org.embulk.input.s3;
 
+import com.amazonaws.AmazonClientException;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
@@ -16,7 +17,9 @@ import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
@@ -55,7 +58,13 @@ public class TestS3InputStreamReopener
         { // retry once
             doThrow(new RuntimeException()).doReturn(s3object("in/aa/a", content)).when(client).getObject(any(GetObjectRequest.class));
 
-            S3InputStreamReopener opener = new S3InputStreamReopener(client, new GetObjectRequest("my_bucket", "in/aa/a"), content.length());
+            S3InputStreamReopener opener = new S3InputStreamReopener(
+                    client,
+                    new GetObjectRequest("my_bucket", "in/aa/a"),
+                    content.length(),
+                    retryExecutor()
+                            .withInitialRetryWait(0)
+                            .withRetryLimit(1));
 
             try (InputStream in = opener.reopen(0, new RuntimeException())) {
                 BufferedReader r = new BufferedReader(new InputStreamReader(in));
@@ -64,6 +73,42 @@ public class TestS3InputStreamReopener
         }
     }
 
+    @Test(expected = AmazonClientException.class)
+    public void reopenS3FileByReopener_on_retry_gave_up_should_throw_original_exception() throws Exception
+    {
+        String content = "value";
+        doThrow(new AmazonClientException("no")).doReturn(s3object("in/aa/a", content)).when(client).getObject(any(GetObjectRequest.class));
+
+        S3InputStreamReopener opener = new S3InputStreamReopener(
+                client,
+                new GetObjectRequest("my_bucket", "in/aa/a"),
+                content.length(),
+                retryExecutor()
+                        .withInitialRetryWait(0)
+                        .withRetryLimit(0));
+
+        opener.reopen(0, new RuntimeException());
+    }
+
+    @Test(expected = AmazonClientException.class)
+    public void reopenS3FileByReopener_on_retry_always_throw_exception()
+            throws Exception
+    {
+        // always failed call with 2 retries
+        doThrow(new AmazonClientException("This exception is thrown when retrying.")).when(client).getObject(any(GetObjectRequest.class));
+        S3InputStreamReopener opener = new S3InputStreamReopener(
+                client,
+                new GetObjectRequest("my_bucket", "in/aa/a"),
+                "value".length(),
+                retryExecutor()
+                        .withInitialRetryWait(0)
+                        .withRetryLimit(2));
+
+        try (InputStream in = opener.reopen(0, new AmazonClientException("This exception can be ignored"))) {
+            fail("Should throw exception.");
+        }
+    }
+
     static S3Object s3object(String key, String value)
     {
         S3Object o = new S3Object();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.16
+  version: 0.2.19
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-11 00:00:00.000000000 Z
+date: 2018-06-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -48,10 +48,13 @@ files:
 - build.gradle
 - lib/embulk/input/s3.rb
 - src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
+- src/main/java/org/embulk/input/s3/AlwaysRetryable.java
 - src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/HttpProxy.java
+- src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
-- src/main/java/org/embulk/input/s3/S3FileInputUtils.java
+- src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java
+- src/test/java/org/embulk/input/s3/TestAlwaysRetryable.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestHttpProxy.java
@@ -62,8 +65,8 @@ files:
 - classpath/aws-java-sdk-kms-1.11.253.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.16.jar
-- classpath/embulk-util-aws-credentials-0.2.16.jar
+- classpath/embulk-input-s3-0.2.19.jar
+- classpath/embulk-util-aws-credentials-0.2.19.jar
 - classpath/httpclient-4.5.2.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/ion-java-1.0.2.jar

data/src/main/java/org/embulk/input/s3/S3FileInputUtils.java

@@ -1,51 +0,0 @@
-package org.embulk.input.s3;
-
-import org.embulk.spi.Exec;
-import org.embulk.spi.util.RetryExecutor;
-import org.slf4j.Logger;
-
-/**
- * Utility class for S3 File Input.
- */
-public final class S3FileInputUtils
-{
-    private S3FileInputUtils()
-    {
-    }
-
-    public static final <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable)
-            throws RetryExecutor.RetryGiveupException, InterruptedException
-    {
-        return RetryExecutor.retryExecutor()
-                .withRetryLimit(maximumRetries)
-                .withInitialRetryWait(initialRetryIntervalMillis)
-                .withMaxRetryWait(maximumRetryIntervalMillis)
-                .runInterruptible(alwaysRetryRetryable);
-    }
-
-    public abstract static class AlwaysRetryRetryable<T> implements RetryExecutor.Retryable<T>
-    {
-        private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
-
-        @Override
-        public abstract T call() throws Exception;
-
-        @Override
-        public boolean isRetryableException(Exception exception)
-        {
-            return true;
-        }
-
-        @Override
-        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
-        {
-            LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
-        }
-
-        @Override
-        public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
-        {
-            LOGGER.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
-        }
-    }
-}