embulk-input-s3 0.3.3 → 0.3.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 889fd9c88e7757c4284247122fd94b65e19f9bb7
-  data.tar.gz: 35ad69c5264debb86359279289135512f5b67ca8
+  metadata.gz: 0497b4779ac08c091c1291583ef439ada4f48ea2
+  data.tar.gz: 0ef8f1d26751cf22d7975b570d9f7cbcfd7e270f
 SHA512:
-  metadata.gz: 1831c5e0392336e86b98a244a8fb45008499ae062308f9dfd9f29d904e6b3c8f5deca5eee68cf2bc5ddcb770d3fbd2a6f65e3f83fbad9e42f41830dea664ffc2
-  data.tar.gz: e24be583cbbc9442056152f9142dbfc413048f8ecbdb0a7dc65b30c0265b5fff80ac324b84598c43f4da71f32857fdcd044c0a7566dca4c23e195ef4618b397a
+  metadata.gz: 82310a7bae6f789ad0962346438a945b4ed59a21fc34be6bbd8e705f979482be58994a1d5d7258f07020ad72cc8dee240313b569b04081f989299a18845dbce5
+  data.tar.gz: 5a41d741bb26cd0d619149c8c1d4b47495c167570f450460e198a129c1b91ebc9f55b45eb771e3b58de6a69bd58f3f52029f0d272200c46f001b3ff6e24ecd13

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -7,15 +7,9 @@ import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.annotations.VisibleForTesting;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -26,6 +20,10 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.input.s3.explorer.S3NameOrderPrefixFileExplorer;
+import org.embulk.input.s3.explorer.S3SingleFileExplorer;
+import org.embulk.input.s3.explorer.S3TimeOrderPrefixFileExplorer;
+import org.embulk.input.s3.utils.DateUtils;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -40,6 +38,9 @@ import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
@@ -51,6 +52,7 @@ public abstract class AbstractS3FileInputPlugin
         implements FileInputPlugin
 {
     private static final Logger LOGGER = Exec.getLogger(S3FileInputPlugin.class);
+    private static final String FULL_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
 
     public interface PluginTask
             extends AwsCredentialsTask, FileList.Task, RetrySupportPluginTask, Task
@@ -88,12 +90,35 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("false")
         boolean getSkipGlacierObjects();
 
+        @Config("use_modified_time")
+        @ConfigDefault("false")
+        boolean getUseModifiedTime();
+
+        @Config("last_modified_time")
+        @ConfigDefault("null")
+        Optional<String> getLastModifiedTime();
+
         // TODO timeout, ssl, etc
 
+        ////////////////////////////////////////
+        // Internal configurations
+        ////////////////////////////////////////
+
         FileList getFiles();
 
         void setFiles(FileList files);
 
+        /**
+         * end_modified_time is conditionally set if modified_time mode is enabled.
+         *
+         * It is internal state and must not be set in config.yml
+         */
+        @Config("__end_modified_time")
+        @ConfigDefault("null")
+        Optional<Date> getEndModifiedTime();
+
+        void setEndModifiedTime(Optional<Date> endModifiedTime);
+
         @ConfigInject
         BufferAllocator getBufferAllocator();
     }
@@ -105,6 +130,7 @@ public abstract class AbstractS3FileInputPlugin
     {
         PluginTask task = config.loadConfig(getTaskClass());
 
+        errorIfInternalParamsAreSet(task);
         validateInputTask(task);
         // list files recursively
         task.setFiles(listFiles(task));
@@ -130,9 +156,15 @@ public abstract class AbstractS3FileInputPlugin
 
         // last_path
         if (task.getIncremental()) {
-            Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
-            LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
-            configDiff.set("last_path", lastPath);
+            if (task.getUseModifiedTime()) {
+                Date endModifiedTime = task.getEndModifiedTime().orElse(new Date());
+                configDiff.set("last_modified_time", new SimpleDateFormat(FULL_DATE_FORMAT).format(endModifiedTime));
+            }
+            else {
+                Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
+                LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
+                configDiff.set("last_path", lastPath);
+            }
         }
         return configDiff;
     }
@@ -237,22 +269,35 @@ public abstract class AbstractS3FileInputPlugin
             String bucketName = task.getBucket();
             FileList.Builder builder = new FileList.Builder(task);
             RetryExecutor retryExec = retryExecutorFrom(task);
+
             if (task.getPath().isPresent()) {
                 LOGGER.info("Start getting object with path: [{}]", task.getPath().get());
-                addS3DirectObject(builder, client, task.getBucket(), task.getPath().get(), retryExec);
+                new S3SingleFileExplorer(bucketName, client, retryExec, task.getPath().get()).addToBuilder(builder);
+                return builder.build();
             }
-            else {
-                // does not need to verify existent path prefix here since there is the validation requires either path or path_prefix
-                LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix().get());
-                if (task.getPathPrefix().get().equals("/")) {
-                    LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
-                }
 
-                listS3FilesByPrefix(builder, client, bucketName,
-                        task.getPathPrefix().get(), task.getLastPath(), task.getSkipGlacierObjects(), retryExec);
-                LOGGER.info("Found total [{}] files", builder.size());
+            // does not need to verify existent path prefix here since there is the validation requires either path or path_prefix
+            LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix().get());
+            if (task.getPathPrefix().get().equals("/")) {
+                LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
             }
 
+            if (task.getUseModifiedTime()) {
+                Date now = new Date();
+                Optional<Date> from = task.getLastModifiedTime().isPresent()
+                        ? Optional.of(DateUtils.parse(task.getLastModifiedTime().get(), Collections.singletonList(FULL_DATE_FORMAT)))
+                        : Optional.empty();
+                task.setEndModifiedTime(Optional.of(now));
+
+                new S3TimeOrderPrefixFileExplorer(bucketName, client, retryExec, task.getPathPrefix().get(),
+                        task.getSkipGlacierObjects(), from, now).addToBuilder(builder);
+            }
+            else {
+                new S3NameOrderPrefixFileExplorer(bucketName, client, retryExec, task.getPathPrefix().get(),
+                        task.getSkipGlacierObjects(), task.getLastPath().orElse(null)).addToBuilder(builder);
+            }
+
+            LOGGER.info("Found total [{}] files", builder.size());
             return builder.build();
         }
         catch (AmazonServiceException ex) {
@@ -268,107 +313,13 @@ public abstract class AbstractS3FileInputPlugin
         }
     }
 
-    @VisibleForTesting
-    public void addS3DirectObject(FileList.Builder builder,
-                                  final AmazonS3 client,
-                                  String bucket,
-                                  String objectKey)
-    {
-        addS3DirectObject(builder, client, bucket, objectKey, null);
-    }
-
-    @VisibleForTesting
-    public void addS3DirectObject(FileList.Builder builder,
-                                   final AmazonS3 client,
-                                   String bucket,
-                                   String objectKey,
-                                   RetryExecutor retryExec)
-    {
-        final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey);
-
-        ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
-            @Override
-            public ObjectMetadata call()
-            {
-                return client.getObjectMetadata(objectMetadataRequest);
-            }
-        }.executeWith(retryExec);
-
-        builder.add(objectKey, objectMetadata.getContentLength());
-    }
-
-    private void validateInputTask(PluginTask task)
+    private void validateInputTask(final PluginTask task)
     {
         if (!task.getPathPrefix().isPresent() && !task.getPath().isPresent()) {
             throw new ConfigException("Either path or path_prefix is required");
         }
     }
 
-    @VisibleForTesting
-    public static void listS3FilesByPrefix(FileList.Builder builder,
-                                           final AmazonS3 client,
-                                           String bucketName,
-                                           String prefix,
-                                           Optional<String> lastPath,
-                                           boolean skipGlacierObjects)
-    {
-        listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null);
-    }
-
-    /**
-     * Lists S3 filenames filtered by prefix.
-     * <p>
-     * The resulting list does not include the file that's size == 0.
-     * @param builder custom Filelist builder
-     * @param client Amazon S3
-     * @param bucketName Amazon S3 bucket name
-     * @param prefix Amazon S3 bucket name prefix
-     * @param lastPath last path
-     * @param skipGlacierObjects skip gracier objects
-     * @param retryExec a retry executor object to do the retrying
-     */
-    @VisibleForTesting
-    public static void listS3FilesByPrefix(FileList.Builder builder,
-                                           final AmazonS3 client,
-                                           String bucketName,
-                                           String prefix,
-                                           Optional<String> lastPath,
-                                           boolean skipGlacierObjects,
-                                           RetryExecutor retryExec)
-    {
-        String lastKey = lastPath.orElse(null);
-        do {
-            final String finalLastKey = lastKey;
-            final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
-            ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects") {
-                @Override
-                public ObjectListing call()
-                {
-                    return client.listObjects(req);
-                }
-            }.executeWith(retryExec);
-            for (S3ObjectSummary s : ol.getObjectSummaries()) {
-                if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
-                    if (skipGlacierObjects) {
-                        Exec.getLogger("AbstractS3FileInputPlugin.class").warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
-                        continue;
-                    }
-                    else {
-                        throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
-                    }
-                }
-                if (s.getSize() > 0) {
-                    builder.add(s.getKey(), s.getSize());
-                    if (!builder.needsMore()) {
-                        LOGGER.warn("Too many files matched, stop listing file");
-                        return;
-                    }
-                }
-            }
-            lastKey = ol.getNextMarker();
-        } while (lastKey != null);
-    }
-
     @Override
     public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
@@ -440,6 +391,14 @@ public abstract class AbstractS3FileInputPlugin
         }
     }
 
+    @VisibleForTesting
+    static void errorIfInternalParamsAreSet(PluginTask task)
+    {
+        if (task.getEndModifiedTime().isPresent()) {
+            throw new ConfigException("'__end_modified_time' must not be set.");
+        }
+    }
+
     // TODO create single-file InputStreamFileInput utility
     private class SingleFileProvider
             implements InputStreamFileInput.Provider

data/src/main/java/org/embulk/input/s3/DefaultRetryable.java

@@ -19,7 +19,7 @@ import static org.embulk.spi.util.RetryExecutor.Retryable;
  * Retryable utility, regardless the occurred exceptions,
  * Also provide a default approach for exception propagation.
  */
-class DefaultRetryable<T> implements Retryable<T>
+public class DefaultRetryable<T> implements Retryable<T>
 {
     private static final Logger log = Exec.getLogger(DefaultRetryable.class);
     private static final Set<Integer> NONRETRYABLE_STATUS_CODES = new HashSet<Integer>(2);

data/src/main/java/org/embulk/input/s3/explorer/S3FileExplorer.java

@@ -0,0 +1,21 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+
+public abstract class S3FileExplorer
+{
+    protected String bucketName;
+    protected AmazonS3 s3Client;
+    protected RetryExecutor retryExecutor;
+
+    public S3FileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor)
+    {
+        this.bucketName = bucketName;
+        this.s3Client = s3Client;
+        this.retryExecutor = retryExecutor;
+    }
+
+    public abstract void addToBuilder(FileList.Builder builder);
+}

data/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java

@@ -0,0 +1,45 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.spi.util.RetryExecutor;
+
+import java.util.List;
+
+public class S3NameOrderPrefixFileExplorer extends S3PrefixFileExplorer
+{
+    private String lastPath;
+
+    public S3NameOrderPrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor,
+            final String pathPrefix, final boolean skipGlacierObjects, final String lastPath)
+    {
+        super(bucketName, s3Client, retryExecutor, pathPrefix, skipGlacierObjects);
+        this.lastPath = lastPath;
+    }
+
+    @Override
+    protected List<S3ObjectSummary> fetch()
+    {
+        final ListObjectsRequest req = new ListObjectsRequest(bucketName, pathPrefix, lastPath, null, 1024);
+        final ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects")
+        {
+            @Override
+            public ObjectListing call()
+            {
+                return s3Client.listObjects(req);
+            }
+        }.executeWith(retryExecutor);
+        lastPath = ol.getNextMarker();
+
+        return ol.getObjectSummaries();
+    }
+
+    @Override
+    protected boolean hasNext()
+    {
+        return lastPath != null;
+    }
+}

data/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java

@@ -0,0 +1,57 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
+import org.embulk.config.ConfigException;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.util.List;
+
+public abstract class S3PrefixFileExplorer extends S3FileExplorer
+{
+    private static final Logger LOGGER = Exec.getLogger(S3PrefixFileExplorer.class);
+
+    protected String pathPrefix;
+
+    private final boolean skipGlacierObjects;
+
+    public S3PrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor, final String pathPrefix, final boolean skipGlacierObjects)
+    {
+        super(bucketName, s3Client, retryExecutor);
+        this.pathPrefix = pathPrefix;
+        this.skipGlacierObjects = skipGlacierObjects;
+    }
+
+    @Override
+    public void addToBuilder(final FileList.Builder builder)
+    {
+        do {
+            final List<S3ObjectSummary> s3ObjectSummaries = fetch();
+
+            for (final S3ObjectSummary s : s3ObjectSummaries) {
+                if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
+                    if (skipGlacierObjects) {
+                        LOGGER.warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
+                        continue;
+                    }
+                    throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
+                }
+                if (s.getSize() > 0) {
+                    builder.add(s.getKey(), s.getSize());
+                    if (!builder.needsMore()) {
+                        LOGGER.warn("Too many files matched, stop listing file");
+                        return;
+                    }
+                }
+            }
+        } while (hasNext());
+    }
+
+    protected abstract List<S3ObjectSummary> fetch();
+
+    protected abstract boolean hasNext();
+}

data/src/main/java/org/embulk/input/s3/explorer/S3SingleFileExplorer.java

@@ -0,0 +1,35 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+
+public class S3SingleFileExplorer extends S3FileExplorer
+{
+    private final String path;
+
+    public S3SingleFileExplorer(final String bucket, final AmazonS3 client, final RetryExecutor retryExecutor, final String path)
+    {
+        super(bucket, client, retryExecutor);
+        this.path = path;
+    }
+
+    @Override
+    public void addToBuilder(final FileList.Builder builder)
+    {
+        final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucketName, path);
+
+        final ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
+            @Override
+            public ObjectMetadata call()
+            {
+                return s3Client.getObjectMetadata(objectMetadataRequest);
+            }
+        }.executeWith(retryExecutor);
+
+        builder.add(path, objectMetadata.getContentLength());
+    }
+}

data/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java

@@ -0,0 +1,70 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class S3TimeOrderPrefixFileExplorer extends S3PrefixFileExplorer
+{
+    private static final Logger LOGGER = Exec.getLogger(S3TimeOrderPrefixFileExplorer.class);
+
+    private final Optional<Date> from;
+    private final Date to;
+
+    private String lastPath;
+
+    private int numOfReq = 0;
+
+    public S3TimeOrderPrefixFileExplorer(final String bucket, final AmazonS3 client, final RetryExecutor retryExecutor,
+            final String pathPrefix, final boolean skipGlacierObjects, final Optional<Date> from, final Date to)
+    {
+        super(bucket, client, retryExecutor, pathPrefix, skipGlacierObjects);
+        this.from = from;
+        this.to = to;
+    }
+
+    @Override
+    public List<S3ObjectSummary> fetch()
+    {
+        ++numOfReq;
+
+        final ListObjectsRequest req = new ListObjectsRequest(bucketName, pathPrefix, lastPath, null, 1024);
+        final ObjectListing objectListing = new DefaultRetryable<ObjectListing>("Listing objects")
+        {
+            @Override
+            public ObjectListing call()
+            {
+                return s3Client.listObjects(req);
+            }
+        }.executeWith(retryExecutor);
+        lastPath = objectListing.getNextMarker();
+
+        return objectListing.getObjectSummaries()
+                .stream()
+                .filter(s3ObjectSummary -> s3ObjectSummary.getLastModified().before(to)
+                        && (!from.isPresent() || s3ObjectSummary.getLastModified().equals(from.get()) || s3ObjectSummary.getLastModified().after(from.get())))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        if (lastPath == null) {
+            LOGGER.info("The total number of LIST requests is {}{}.", numOfReq,
+                    numOfReq < 10 ? StringUtils.EMPTY : ". Clean up your s3 bucket to reduce the number of requests and improve the ingesting performance");
+            return false;
+        }
+        return true;
+    }
+}

data/src/main/java/org/embulk/input/s3/utils/DateUtils.java

@@ -0,0 +1,28 @@
+package org.embulk.input.s3.utils;
+
+import com.google.common.base.Joiner;
+import org.embulk.config.ConfigException;
+import org.joda.time.format.DateTimeFormat;
+
+import java.util.Date;
+import java.util.List;
+
+public class DateUtils
+{
+    public static Date parse(final String value, final List<String> supportedFormats)
+            throws ConfigException
+    {
+        for (final String fmt : supportedFormats) {
+            try {
+                return DateTimeFormat.forPattern(fmt).parseDateTime(value).toDate();
+            } catch (final IllegalArgumentException e) {
+                // ignorable exception
+            }
+        }
+        throw new ConfigException("Unsupported DateTime value: '" + value + "', supported formats: [" + Joiner.on(",").join(supportedFormats) + "]");
+    }
+
+    private DateUtils()
+    {
+    }
+}

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -1,16 +1,11 @@
 package org.embulk.input.s3;
 
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.Region;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -25,21 +20,15 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
-import org.mockito.Mockito;
 
-import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 
 import static org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assume.assumeNotNull;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
 
 public class TestS3FileInputPlugin
 {
@@ -97,7 +86,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void useLastPath()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy().set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv");
         ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
@@ -117,7 +105,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void emptyFilesWithLastPath()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy()
                 .set("path_prefix", "empty_files_prefix")
@@ -130,7 +117,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void useTotalFileCountLimit()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy().set("total_file_count_limit", 0);
         ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
@@ -141,7 +127,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void usePathMatchPattern()
-            throws Exception
     {
         { // match pattern
             ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/sample_01");
@@ -227,44 +212,6 @@ public class TestS3FileInputPlugin
         assertEquals(s3Client.getRegion(), Region.US_Standard);
     }
 
-    @Test(expected = ConfigException.class)
-    public void useSkipGlacierObjects() throws Exception
-    {
-        AmazonS3 client;
-        client = mock(AmazonS3.class);
-        doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class));
-
-        AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS);
-        plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.empty(), false);
-    }
-
-    private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize)
-    {
-        FileList.Builder builder = new FileList.Builder(config);
-        for (int i = 0; i < nameAndSize.length; i += 2) {
-            builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
-        }
-        return builder;
-    }
-
-    private ObjectListing s3objectList(String key, StorageClass storageClass) throws Exception
-    {
-        ObjectListing list = new ObjectListing();
-
-        S3ObjectSummary element = new S3ObjectSummary();
-        element.setKey(key);
-        element.setStorageClass(storageClass.toString());
-
-        List<S3ObjectSummary> objectSummaries = new ArrayList<>();
-        objectSummaries.add(element);
-
-        Field field = list.getClass().getDeclaredField("objectSummaries");
-        field.setAccessible(true);
-        field.set(list, objectSummaries);
-
-        return list;
-    }
-
     static class Control
             implements InputPlugin.Control
     {

data/src/test/java/org/embulk/input/s3/explorer/TestS3NameOrderPrefixFileExplorer.java

@@ -0,0 +1,67 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.internal.util.reflection.FieldSetter;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3NameOrderPrefixFileExplorer
+{
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String PATH_PREFIX = "path_prefix";
+    private static final String LAST_PATH = "last_path";
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    private S3NameOrderPrefixFileExplorer s3NameOrderPrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3NameOrderPrefixFileExplorer = new S3NameOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, false, LAST_PATH);
+    }
+
+    @Test
+    public void fetch_should_return_list_objects()
+    {
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+
+        s3NameOrderPrefixFileExplorer.fetch();
+        final ArgumentCaptor<ListObjectsRequest> listObjectsRequestCaptor = ArgumentCaptor.forClass(ListObjectsRequest.class);
+
+        verify(ol).getNextMarker();
+        verify(s3Client).listObjects(listObjectsRequestCaptor.capture());
+        final ListObjectsRequest listObjectsRequest = listObjectsRequestCaptor.getValue();
+        assertEquals(BUCKET_NAME, listObjectsRequest.getBucketName());
+        assertEquals(PATH_PREFIX, listObjectsRequest.getPrefix());
+        assertEquals(LAST_PATH, listObjectsRequest.getMarker());
+    }
+
+    @Test
+    public void hasNext_should_return_false_if_no_lastpath() throws NoSuchFieldException
+    {
+        new FieldSetter(s3NameOrderPrefixFileExplorer, s3NameOrderPrefixFileExplorer.getClass().getDeclaredField("lastPath")).set(null);
+        assertFalse(s3NameOrderPrefixFileExplorer.hasNext());
+    }
+}

data/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java

@@ -0,0 +1,128 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3PrefixFileExplorer
+{
+    private static final String PATH_PREFIX = "path_prefix";
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String OBJECT_KEY = "key";
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    @Mock
+    private FileList.Builder builder;
+
+    @Mock
+    private S3ObjectSummary s3ObjectSummary;
+
+    private S3PrefixFileExplorer s3PrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, false);
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+    }
+
+    @Test(expected = ConfigException.class)
+    public void addToBuilder_should_throw_exception_if_notskipped_glacier_storage()
+    {
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
+        s3PrefixFileExplorer.addToBuilder(builder);
+    }
+
+    @Test
+    public void addToBuilder_should_skip_glacier_storage_if_allowed()
+    {
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
+        // override spied object for changing `skipGlacierObjects`
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, true);
+        doReturn(false).when(s3PrefixFileExplorer).hasNext();
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+        s3PrefixFileExplorer.addToBuilder(builder);
+
+        verify(s3PrefixFileExplorer).hasNext();
+        verify(s3ObjectSummary, never()).getSize();
+    }
+
+    @Test
+    public void addToBuilder_should_loop_till_nothing_left()
+    {
+        // There are 3 loops totally but only 2 keys have been imported because the first key is in Glacier storage class and is skipped
+        when(builder.needsMore()).thenReturn(true);
+        // override spied object for changing `skipGlacierObjects`
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, true);
+        when(s3ObjectSummary.getStorageClass())
+                .thenReturn(StorageClass.Glacier.toString())
+                .thenReturn(StorageClass.Standard.toString());
+        when(s3ObjectSummary.getSize()).thenReturn(1L);
+        when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+        doReturn(true).doReturn(true).doReturn(false).when(s3PrefixFileExplorer).hasNext();
+
+        s3PrefixFileExplorer.addToBuilder(builder);
+        verify(builder, times(2)).add(PATH_PREFIX + OBJECT_KEY, 1);
+    }
+
+    @Test
+    public void addToBuilder_should_stop_import_if_too_many_files()
+    {
+        when(builder.needsMore()).thenReturn(false);
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
+        when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
+        when(s3ObjectSummary.getSize()).thenReturn(1L);
+        doReturn(true).when(s3PrefixFileExplorer).hasNext();
+        s3PrefixFileExplorer.addToBuilder(builder);
+
+        verify(builder).add(PATH_PREFIX + OBJECT_KEY, 1);
+        verify(s3PrefixFileExplorer, never()).hasNext();
+    }
+
+    private S3PrefixFileExplorer spyS3PrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor, final String pathPrefix, final boolean skipGlacierObjects)
+    {
+        return spy(new S3PrefixFileExplorer(bucketName, s3Client, retryExecutor, pathPrefix, skipGlacierObjects)
+        {
+            @Override
+            protected List<S3ObjectSummary> fetch()
+            {
+                return null;
+            }
+
+            @Override
+            protected boolean hasNext()
+            {
+                return false;
+            }
+        });
+    }
+}

data/src/test/java/org/embulk/input/s3/explorer/TestS3SingleFileExplorer.java

@@ -0,0 +1,56 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.s3.FileList;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3SingleFileExplorer
+{
+    private static final String PATH = "path";
+    private static final String BUCKET_NAME = "bucket_name";
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    @Mock
+    private FileList.Builder builder;
+
+    @Mock
+    private ObjectMetadata metadata;
+
+    private S3SingleFileExplorer s3SingleFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3SingleFileExplorer = new S3SingleFileExplorer(BUCKET_NAME, s3Client, null, PATH);
+    }
+
+    @Test
+    public void addToBuilder_should_request_single_object_metadata()
+    {
+        when(s3Client.getObjectMetadata(any(GetObjectMetadataRequest.class))).thenReturn(metadata);
+        when(metadata.getContentLength()).thenReturn(1L);
+        s3SingleFileExplorer.addToBuilder(builder);
+
+        verify(builder).add(PATH, 1);
+    }
+}

data/src/test/java/org/embulk/input/s3/explorer/TestS3TimeOrderPrefixFileExplorer.java

@@ -0,0 +1,112 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.internal.util.reflection.FieldSetter;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3TimeOrderPrefixFileExplorer
+{
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String PATH_PREFIX = "path_prefix";
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    private S3TimeOrderPrefixFileExplorer s3TimeOrderPrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        final Calendar cal = Calendar.getInstance();
+        cal.set(2019, Calendar.MAY, 25, 10, 0);
+        s3TimeOrderPrefixFileExplorer = new S3TimeOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX,
+                false, Optional.empty(), cal.getTime());
+    }
+
+    @Test
+    public void fetch_should_return_filtered_objects_before_end_time()
+    {
+        final S3ObjectSummary s3ObjectBefore = mock(S3ObjectSummary.class);
+        final Calendar cal = Calendar.getInstance();
+        cal.set(2019, Calendar.MAY, 24, 10, 0);
+        when(s3ObjectBefore.getLastModified()).thenReturn(cal.getTime());
+
+        final S3ObjectSummary s3ObjectAfter = mock(S3ObjectSummary.class);
+        cal.set(2019, Calendar.MAY, 26, 10, 0);
+        when(s3ObjectAfter.getLastModified()).thenReturn(cal.getTime());
+
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+        when(ol.getObjectSummaries()).thenReturn(Arrays.asList(s3ObjectBefore, s3ObjectAfter));
+
+        final List<S3ObjectSummary> result = s3TimeOrderPrefixFileExplorer.fetch();
+        assertEquals(1, result.size());
+        assertEquals(s3ObjectBefore, result.get(0));
+    }
+
+    @Test
+    public void fetch_should_return_filtered_objects_after_or_equals_begin_time()
+    {
+        final Calendar to = Calendar.getInstance();
+        to.set(2019, Calendar.MAY, 25, 10, 0);
+        final Calendar from = Calendar.getInstance();
+        from.set(2019, Calendar.MAY, 24, 10, 0);
+        s3TimeOrderPrefixFileExplorer = new S3TimeOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX,
+                false, Optional.of(from.getTime()), to.getTime());
+
+        final S3ObjectSummary s3ObjectEqual = mock(S3ObjectSummary.class);
+        final Calendar equalCal = Calendar.getInstance();
+        equalCal.set(2019, Calendar.MAY, 24, 10, 0);
+        when(s3ObjectEqual.getLastModified()).thenReturn(equalCal.getTime());
+
+        final S3ObjectSummary s3ObjectBefore = mock(S3ObjectSummary.class);
+        final Calendar beforeCal = Calendar.getInstance();
+        beforeCal.set(2019, Calendar.MAY, 24, 20, 0);
+        when(s3ObjectBefore.getLastModified()).thenReturn(beforeCal.getTime());
+
+        final S3ObjectSummary s3ObjectAfter = mock(S3ObjectSummary.class);
+        final Calendar afterCal = Calendar.getInstance();
+        afterCal.set(2019, Calendar.MAY, 26, 10, 0);
+        when(s3ObjectAfter.getLastModified()).thenReturn(afterCal.getTime());
+
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+        when(ol.getObjectSummaries()).thenReturn(Arrays.asList(s3ObjectEqual, s3ObjectBefore, s3ObjectAfter));
+
+        final List<S3ObjectSummary> result = s3TimeOrderPrefixFileExplorer.fetch();
+        assertEquals(2, result.size());
+        assertEquals(s3ObjectEqual, result.get(0));
+        assertEquals(s3ObjectBefore, result.get(1));
+    }
+
+    @Test
+    public void hasNext_should_return_false_if_no_lastpath() throws NoSuchFieldException
+    {
+        new FieldSetter(s3TimeOrderPrefixFileExplorer, s3TimeOrderPrefixFileExplorer.getClass().getDeclaredField("lastPath")).set(null);
+        assertFalse(s3TimeOrderPrefixFileExplorer.hasNext());
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.3
+  version: 0.3.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-02-05 00:00:00.000000000 Z
+date: 2019-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -53,19 +53,28 @@ files:
 - src/main/java/org/embulk/input/s3/HttpProxy.java
 - src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
-- src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java
+- src/main/java/org/embulk/input/s3/explorer/S3FileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3SingleFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/utils/DateUtils.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestDefaultRetryable.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestHttpProxy.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3NameOrderPrefixFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3SingleFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3TimeOrderPrefixFileExplorer.java
 - src/test/resources/sample_01.csv
-- classpath/embulk-util-aws-credentials-0.3.3.jar
+- classpath/embulk-util-aws-credentials-0.3.4.jar
 - classpath/httpcore-4.4.9.jar
 - classpath/httpclient-4.5.5.jar
 - classpath/ion-java-1.0.2.jar
-- classpath/embulk-input-s3-0.3.3.jar
+- classpath/embulk-input-s3-0.3.4.jar
 - classpath/aws-java-sdk-core-1.11.466.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
 - classpath/commons-codec-1.10.jar

data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java

@@ -1,164 +0,0 @@
-package org.embulk.input.s3;
-
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.http.HttpStatus;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.spi.util.RetryExecutor;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.util.Optional;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-
-public class TestAbstractS3FileInputPlugin
-{
-    private static RetryExecutor retryExecutor()
-    {
-        return RetryExecutor.retryExecutor()
-                .withInitialRetryWait(0)
-                .withMaxRetryWait(0);
-    }
-
-    private static AbstractS3FileInputPlugin dummyS3Plugin()
-    {
-        return new AbstractS3FileInputPlugin()
-        {
-            @Override
-            protected Class<? extends PluginTask> getTaskClass()
-            {
-                return PluginTask.class;
-            }
-        };
-    }
-
-    private static class SomeException extends RuntimeException
-    {
-    }
-
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private AmazonS3 client;
-
-    @Before
-    public void createResources()
-    {
-        client = mock(AmazonS3.class);
-    }
-
-    @Test
-    public void listS3FilesByPrefix()
-    {
-        doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true);
-    }
-
-    @Test
-    public void listS3FileByPrefix_with_retry()
-    {
-        doThrow(new RuntimeException()).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = SomeException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception()
-    {
-        doThrow(new SomeException()).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(0));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("Forbidden exception");
-        exception.setStatusCode(HttpStatus.SC_FORBIDDEN);
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("method not allow exception");
-        exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED);
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("expired token exception");
-        exception.setStatusCode(HttpStatus.SC_BAD_REQUEST);
-        exception.setErrorCode("ExpiredToken");
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test
-    public void addS3DirectObject()
-    {
-        doReturn(new ObjectMetadata()).when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(builder, client, "some_bucket", "some_prefix");
-    }
-
-    @Test
-    public void addS3DirectObject_with_retry()
-    {
-        doThrow(new RuntimeException()).doReturn(new ObjectMetadata())
-                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(
-                builder, client, "some_bucket", "some_prefix",
-                retryExecutor());
-    }
-
-    @Test(expected = SomeException.class)
-    public void addS3DirectObject_on_retry_gave_up_should_throw_original_exception()
-    {
-        doThrow(new SomeException()).doReturn(new ObjectMetadata())
-                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(
-                builder, client, "some_bucket", "some_prefix",
-                retryExecutor().withRetryLimit(0));
-    }
-}