embulk-input-s3 0.2.15 → 0.2.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6efa3b427ccee34a13fae779d053ea5863a676e7
-  data.tar.gz: a54eec2a80f7003d4a7bdeff529e152fe8a7ed3b
+  metadata.gz: 95a2e3a70de76cdd69c0b0252f37b32671ce3896
+  data.tar.gz: 0f7a8a710a28478d528df8a31d08b996a0595772
 SHA512:
-  metadata.gz: 35d00e37563f72c184f1bf9f02186927975a14d9eae0bf2ea240fe31c94b28414b70c1574494be19833305a9908e72081a1b5356da570b48d85106366666cdd3
-  data.tar.gz: 7de346f325126fbc1d1ebd5ca7f36269945bc5d7e9227d4b6bfb88f7b0c56327ed2c95b902377b5c84e297898b9e98a6f14021b0634a0da8c18aed23f8d96c63
+  metadata.gz: 6ae53adc0606da4ec1e2a72a742b18b27ea40145b27baf5375e905670f243b411d43e15a55fdeb2172cc158e1e83b257f290a79d2ff92aa890f9ab57e38d2174
+  data.tar.gz: 332d2963cfde7edf02ed80b9fda7d651a12c490e03f7f2984cee329858b978ebd5c2d5401e9ea30efc5e4a55e66ccd6d87d285a908bd3a4819dffa50de52591e

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -69,6 +69,7 @@ public abstract class AbstractS3FileInputPlugin
         @Config("http_proxy")
         @ConfigDefault("null")
         public Optional<HttpProxy> getHttpProxy();
+
         public void setHttpProxy(Optional<HttpProxy> httpProxy);
 
         @Config("incremental")
@@ -82,6 +83,7 @@ public abstract class AbstractS3FileInputPlugin
         // TODO timeout, ssl, etc
 
         public FileList getFiles();
+
         public void setFiles(FileList files);
 
         @ConfigInject
@@ -104,8 +106,8 @@ public abstract class AbstractS3FileInputPlugin
 
     @Override
     public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control)
+                             int taskCount,
+                             FileInputPlugin.Control control)
     {
         PluginTask task = taskSource.loadTask(getTaskClass());
 
@@ -128,8 +130,8 @@ public abstract class AbstractS3FileInputPlugin
 
     @Override
     public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports)
+                        int taskCount,
+                        List<TaskReport> successTaskReports)
     {
         // do nothing
     }
@@ -139,13 +141,19 @@ public abstract class AbstractS3FileInputPlugin
      * Since this returns an immutable object, it is not for any further customizations by mutating,
      * e.g., {@link AmazonS3#setEndpoint} will throw a runtime {@link UnsupportedOperationException}
      * Subclass's customization should be done through {@link AbstractS3FileInputPlugin#defaultS3ClientBuilder}.
+     * @param task Embulk plugin task
+     * @return AmazonS3
      */
     protected AmazonS3 newS3Client(PluginTask task)
     {
         return defaultS3ClientBuilder(task).build();
     }
 
-    /** A base builder for the subclasses to then customize. */
+    /**
+     * A base builder for the subclasses to then customize.builder
+     * @param task Embulk plugin
+     * @return AmazonS3 client b
+     **/
     protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task)
     {
         return AmazonS3ClientBuilder
@@ -228,21 +236,45 @@ public abstract class AbstractS3FileInputPlugin
             }
             throw ex;
         }
+        catch (InterruptedException | RetryGiveupException ex) {
+            throw new RuntimeException(ex);
+        }
     }
 
     /**
      * Lists S3 filenames filtered by prefix.
-     *
+     * <p>
      * The resulting list does not include the file that's size == 0.
+     * @param builder custom Filelist builder
+     * @param client Amazon S3
+     * @param bucketName Amazon S3 bucket name
+     * @param prefix Amazon S3 bucket name prefix
+     * @param lastPath last path
+     * @param skipGlacierObjects skip gracier objects
+     * @throws RetryGiveupException error when retrying
+     * @throws InterruptedException error when retrying
      */
     public static void listS3FilesByPrefix(FileList.Builder builder,
-            AmazonS3 client, String bucketName,
-            String prefix, Optional<String> lastPath, boolean skipGlacierObjects)
+                                           final AmazonS3 client, final String bucketName,
+                                           final String prefix, Optional<String> lastPath, boolean skipGlacierObjects) throws RetryGiveupException, InterruptedException
     {
         String lastKey = lastPath.orNull();
         do {
-            ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
-            ObjectListing ol = client.listObjects(req);
+            final String finalLastKey = lastKey;
+            Optional<ObjectListing> optOl = S3FileInputUtils.executeWithRetry(3, 500, 30 * 1000, new S3FileInputUtils.AlwaysRetryRetryable<Optional<ObjectListing>>()
+            {
+                @Override
+                public Optional<ObjectListing> call() throws AmazonServiceException
+                {
+                    ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
+                    ObjectListing ol = client.listObjects(req);
+                    return Optional.of(ol);
+                }
+            });
+            if (!optOl.isPresent()) {
+                break;
+            }
+            ObjectListing ol = optOl.get();
             for (S3ObjectSummary s : ol.getObjectSummaries()) {
                 if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
                     if (skipGlacierObjects) {
@@ -262,7 +294,7 @@ public abstract class AbstractS3FileInputPlugin
                 }
             }
             lastKey = ol.getNextMarker();
-        } while(lastKey != null);
+        } while (lastKey != null);
     }
 
     @Override
@@ -294,45 +326,46 @@ public abstract class AbstractS3FileInputPlugin
         {
             try {
                 return retryExecutor()
-                    .withRetryLimit(3)
-                    .withInitialRetryWait(500)
-                    .withMaxRetryWait(30 * 1000)
-                    .runInterruptible(new Retryable<InputStream>() {
-                        @Override
-                        public InputStream call() throws InterruptedIOException
+                        .withRetryLimit(3)
+                        .withInitialRetryWait(500)
+                        .withMaxRetryWait(30 * 1000)
+                        .runInterruptible(new Retryable<InputStream>()
                         {
-                            log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
-                            request.setRange(offset, contentLength - 1);  // [first, last]
-                            return client.getObject(request).getObjectContent();
-                        }
-
-                        @Override
-                        public boolean isRetryableException(Exception exception)
-                        {
-                            return true;  // TODO
-                        }
+                            @Override
+                            public InputStream call() throws InterruptedIOException
+                            {
+                                log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+                                request.setRange(offset, contentLength - 1);  // [first, last]
+                                return client.getObject(request).getObjectContent();
+                            }
 
-                        @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                                throws RetryGiveupException
-                        {
-                            String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
-                            if (retryCount % 3 == 0) {
-                                log.warn(message, exception);
+                            @Override
+                            public boolean isRetryableException(Exception exception)
+                            {
+                                return true;  // TODO
                             }
-                            else {
-                                log.warn(message);
+
+                            @Override
+                            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                    throws RetryGiveupException
+                            {
+                                String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                        retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                                if (retryCount % 3 == 0) {
+                                    log.warn(message, exception);
+                                }
+                                else {
+                                    log.warn(message);
+                                }
                             }
-                        }
 
-                        @Override
-                        public void onGiveup(Exception firstException, Exception lastException)
-                                throws RetryGiveupException
-                        {
-                            log.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
-                        }
-                    });
+                            @Override
+                            public void onGiveup(Exception firstException, Exception lastException)
+                                    throws RetryGiveupException
+                            {
+                                log.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
+                            }
+                        });
             }
             catch (RetryGiveupException ex) {
                 Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);

data/src/main/java/org/embulk/input/s3/S3FileInputUtils.java

@@ -0,0 +1,51 @@
+package org.embulk.input.s3;
+
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+/**
+ * Utility class for S3 File Input.
+ */
+public final class S3FileInputUtils
+{
+    private S3FileInputUtils()
+    {
+    }
+
+    public static final <T> T executeWithRetry(int maximumRetries, int initialRetryIntervalMillis, int maximumRetryIntervalMillis, AlwaysRetryRetryable<T> alwaysRetryRetryable)
+            throws RetryExecutor.RetryGiveupException, InterruptedException
+    {
+        return RetryExecutor.retryExecutor()
+                .withRetryLimit(maximumRetries)
+                .withInitialRetryWait(initialRetryIntervalMillis)
+                .withMaxRetryWait(maximumRetryIntervalMillis)
+                .runInterruptible(alwaysRetryRetryable);
+    }
+
+    public abstract static class AlwaysRetryRetryable<T> implements RetryExecutor.Retryable<T>
+    {
+        private static final Logger LOGGER = Exec.getLogger(AlwaysRetryRetryable.class);
+
+        @Override
+        public abstract T call() throws Exception;
+
+        @Override
+        public boolean isRetryableException(Exception exception)
+        {
+            return true;
+        }
+
+        @Override
+        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.info("Retry [{}]/[{}] with retryWait [{}] on exception {}", retryCount, retryLimit, retryWait, exception.getMessage());
+        }
+
+        @Override
+        public void onGiveup(Exception firstException, Exception lastException) throws RetryExecutor.RetryGiveupException
+        {
+            LOGGER.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.15
+  version: 0.2.16
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-07 00:00:00.000000000 Z
+date: 2018-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +51,7 @@ files:
 - src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/HttpProxy.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
+- src/main/java/org/embulk/input/s3/S3FileInputUtils.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestHttpProxy.java
@@ -61,8 +62,8 @@ files:
 - classpath/aws-java-sdk-kms-1.11.253.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.15.jar
-- classpath/embulk-util-aws-credentials-0.2.15.jar
+- classpath/embulk-input-s3-0.2.16.jar
+- classpath/embulk-util-aws-credentials-0.2.16.jar
 - classpath/httpclient-4.5.2.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/ion-java-1.0.2.jar