embulk-input-s3 0.2.19 → 0.2.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3c9e972f5326281de40fc46190dc3d268282b34e
-  data.tar.gz: 0c63221f881710e6c4a18dbf10ab73be701193ff
+  metadata.gz: 4bd7ca7dab8a4d3a6f25af6a33cf367965693e30
+  data.tar.gz: 4d3a8175e1d6ce32eaa780a44d8de66849ae34ca
 SHA512:
-  metadata.gz: 3983830d6e700a3b3e0a2cdadd063613c4ff41b39e01c96c00c2f8a33515fd8d8be601c644184119bb87e46d1eab394869e91cf1c557c6420ef022f40a93b31e
-  data.tar.gz: 27471ec5bbc698865ab44d2d4ca94b2074c7589c5c9c0ce919b3c3b5bc18b885b6566b48fb74803c99abc10c04716771a1b9faa231ff780de3740da55b468358
+  metadata.gz: e6aecfe225ac14c708a5f6acd7a605be5bca343f33af5379e1fa0b4a33219cf342f65976667d943b90da6c812f55a6f96680659db39752a32d91c1e3abeec496
+  data.tar.gz: 300e89205477ae9d01122fdda478b777b11605eb0302a9f2afee8ee536ef2a31b76f1001572800eca4f905452f8b894aad0bf6f3555bf5783e4bd816c3ce3151

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -4,6 +4,7 @@ import com.amazonaws.AmazonServiceException;
 import com.amazonaws.ClientConfiguration;
 import com.amazonaws.Protocol;
 import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
@@ -177,11 +178,12 @@ public abstract class AbstractS3FileInputPlugin
     {
         ClientConfiguration clientConfig = new ClientConfiguration();
 
+        /** PLT-9886: disable built-in retry*/
         //clientConfig.setProtocol(Protocol.HTTP);
-        clientConfig.setMaxConnections(50); // SDK default: 50
-        clientConfig.setMaxErrorRetry(3); // SDK default: 3
-        clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
-
+//        clientConfig.setMaxConnections(50); // SDK default: 50
+//        clientConfig.setMaxErrorRetry(3); // SDK default: 3
+//        clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
+        clientConfig.setRetryPolicy(PredefinedRetryPolicies.NO_RETRY_POLICY);
         // set http proxy
         if (task.getHttpProxy().isPresent()) {
             setHttpProxyInAwsClient(clientConfig, task.getHttpProxy().get());
@@ -283,7 +285,7 @@ public abstract class AbstractS3FileInputPlugin
     {
         final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey);
 
-        ObjectMetadata objectMetadata = new AlwaysRetryable<ObjectMetadata>("Looking up for a single object") {
+        ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
             @Override
             public ObjectMetadata call()
             {
@@ -337,7 +339,7 @@ public abstract class AbstractS3FileInputPlugin
         do {
             final String finalLastKey = lastKey;
             final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
-            ObjectListing ol = new AlwaysRetryable<ObjectListing>("Listing objects") {
+            ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects") {
                 @Override
                 public ObjectListing call()
                 {
@@ -403,7 +405,7 @@ public abstract class AbstractS3FileInputPlugin
             log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
             request.setRange(offset, contentLength - 1);  // [first, last]
 
-            return new AlwaysRetryable<S3ObjectInputStream>("Opening the file") {
+            return new DefaultRetryable<S3ObjectInputStream>("Opening the file") {
                 @Override
                 public S3ObjectInputStream call()
                 {

data/src/main/java/org/embulk/input/s3/{AlwaysRetryable.java → DefaultRetryable.java}

@@ -1,10 +1,14 @@
 package org.embulk.input.s3;
 
+import com.amazonaws.AmazonServiceException;
 import com.google.common.base.Throwables;
+import org.apache.http.HttpStatus;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.RetryExecutor;
 import org.slf4j.Logger;
 
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.Callable;
 
 import static java.lang.String.format;
@@ -12,20 +16,27 @@ import static org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import static org.embulk.spi.util.RetryExecutor.Retryable;
 
 /**
- * Always retry, regardless the occurred exceptions,
+ * Retryable utility, regardless the occurred exceptions,
  * Also provide a default approach for exception propagation.
  */
-class AlwaysRetryable<T> implements Retryable<T>
+class DefaultRetryable<T> implements Retryable<T>
 {
-    private static final Logger log = Exec.getLogger(AlwaysRetryable.class);
-
+    private static final Logger log = Exec.getLogger(DefaultRetryable.class);
+    private static final Set<Integer> NONRETRYABLE_STATUS_CODES = new HashSet<Integer>(2);
+    private static final Set<String> NONRETRYABLE_ERROR_CODES = new HashSet<String>(1);
     private String operationName;
     private Callable<T> callable;
 
+    static {
+        NONRETRYABLE_STATUS_CODES.add(HttpStatus.SC_FORBIDDEN);
+        NONRETRYABLE_STATUS_CODES.add(HttpStatus.SC_METHOD_NOT_ALLOWED);
+        NONRETRYABLE_ERROR_CODES.add("ExpiredToken");
+    }
+
     /**
      * @param operationName the name that will be referred on logging
      */
-    public AlwaysRetryable(String operationName)
+    public DefaultRetryable(String operationName)
     {
         this.operationName = operationName;
     }
@@ -34,18 +45,18 @@ class AlwaysRetryable<T> implements Retryable<T>
      * @param operationName the name that will be referred on logging
      * @param callable the operation, either define this at construction time or override the call() method
      */
-    public AlwaysRetryable(String operationName, Callable<T> callable)
+    public DefaultRetryable(String operationName, Callable<T> callable)
     {
         this.operationName = operationName;
         this.callable = callable;
     }
 
-    public AlwaysRetryable()
+    public DefaultRetryable()
     {
         this("Anonymous operation");
     }
 
-    public AlwaysRetryable(Callable<T> callable)
+    public DefaultRetryable(Callable<T> callable)
     {
         this("Anonymous operation", callable);
     }
@@ -64,6 +75,11 @@ class AlwaysRetryable<T> implements Retryable<T>
     @Override
     public boolean isRetryableException(Exception exception)
     {
+        // No retry on a subset of service exceptions
+        if (exception instanceof AmazonServiceException) {
+            AmazonServiceException ase = (AmazonServiceException) exception;
+            return !NONRETRYABLE_STATUS_CODES.contains(ase.getStatusCode()) && !NONRETRYABLE_ERROR_CODES.contains(ase.getErrorCode());
+        }
         return true;
     }
 

data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java

@@ -1,11 +1,13 @@
 package org.embulk.input.s3;
 
+import com.amazonaws.AmazonServiceException;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.google.common.base.Optional;
+import org.apache.http.HttpStatus;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.spi.util.RetryExecutor;
 import org.junit.Before;
@@ -83,6 +85,52 @@ public class TestAbstractS3FileInputPlugin
                 retryExecutor().withRetryLimit(0));
     }
 
+    @Test(expected = AmazonServiceException.class)
+    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code()
+    {
+        AmazonServiceException exception = new AmazonServiceException("Forbidden exception");
+        exception.setStatusCode(HttpStatus.SC_FORBIDDEN);
+        exception.setErrorType(AmazonServiceException.ErrorType.Client);
+
+        doThrow(exception).doReturn(new ObjectListing())
+                .when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(
+                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
+                retryExecutor().withRetryLimit(1));
+    }
+
+    @Test(expected = AmazonServiceException.class)
+    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code()
+    {
+        AmazonServiceException exception = new AmazonServiceException("method not allow exception");
+        exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED);
+        exception.setErrorType(AmazonServiceException.ErrorType.Client);
+
+        doThrow(exception).doReturn(new ObjectListing())
+                .when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(
+                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
+                retryExecutor().withRetryLimit(1));
+    }
+
+    @Test(expected = AmazonServiceException.class)
+    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code()
+    {
+        AmazonServiceException exception = new AmazonServiceException("expired token exception");
+        exception.setStatusCode(HttpStatus.SC_BAD_REQUEST);
+        exception.setErrorCode("ExpiredToken");
+        exception.setErrorType(AmazonServiceException.ErrorType.Client);
+
+        doThrow(exception).doReturn(new ObjectListing())
+                .when(client).listObjects(any(ListObjectsRequest.class));
+        FileList.Builder builder = new FileList.Builder();
+        dummyS3Plugin().listS3FilesByPrefix(
+                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
+                retryExecutor().withRetryLimit(1));
+    }
+
     @Test
     public void addS3DirectObject()
     {

data/src/test/java/org/embulk/input/s3/{TestAlwaysRetryable.java → TestDefaultRetryable.java}

@@ -12,10 +12,10 @@ import java.util.concurrent.Callable;
 import static java.lang.String.format;
 import static org.msgpack.core.Preconditions.checkArgument;
 
-public class TestAlwaysRetryable
+public class TestDefaultRetryable
 {
     @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();  // require for AlwaysRetryable's logger
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();  // require for DefaultRetryable's logger
 
     private static class Deny extends RuntimeException implements Callable
     {
@@ -71,16 +71,16 @@ public class TestAlwaysRetryable
     {
         retryExecutor()
                 .withRetryLimit(0)
-                .run(new AlwaysRetryable(Deny.until(0)));
+                .run(new DefaultRetryable(Deny.until(0)));
         retryExecutor()
                 .withRetryLimit(1)
-                .run(new AlwaysRetryable(Deny.until(1)));
+                .run(new DefaultRetryable(Deny.until(1)));
         retryExecutor()
                 .withRetryLimit(2)
-                .run(new AlwaysRetryable(Deny.until(1)));
+                .run(new DefaultRetryable(Deny.until(1)));
         retryExecutor()
                 .withRetryLimit(3)
-                .run(new AlwaysRetryable(Deny.until(2)));
+                .run(new DefaultRetryable(Deny.until(2)));
     }
 
     @Test(expected = RetryGiveupException.class)
@@ -89,14 +89,14 @@ public class TestAlwaysRetryable
     {
         retryExecutor()
                 .withRetryLimit(3)
-                .run(new AlwaysRetryable(Deny.until(4)));
+                .run(new DefaultRetryable(Deny.until(4)));
     }
 
     @Test(expected = Deny.class)
     @SuppressWarnings("unchecked")
     public void execute_should_unwrap_RetryGiveupException() throws Exception
     {
-        new AlwaysRetryable(Deny.until(4))
+        new DefaultRetryable(Deny.until(4))
                 .executeWith(retryExecutor().withRetryLimit(3));
     }
 
@@ -104,7 +104,7 @@ public class TestAlwaysRetryable
     @SuppressWarnings("unchecked")
     public void execute_should_unwrap_RetryGiveupException_but_rewrap_checked_exception_in_a_RuntimeException()
     {
-        new AlwaysRetryable(Deny.until(4).with(new Exception("A checked exception")))
+        new DefaultRetryable(Deny.until(4).with(new Exception("A checked exception")))
                 .executeWith(retryExecutor().withRetryLimit(3));
     }
 
@@ -114,7 +114,7 @@ public class TestAlwaysRetryable
         RetryExecutor retryExc = retryExecutor().withRetryLimit(3);
         // An explicit type parameter for operation return type is needed here,
         // Without one, javac (at least on 1.8) will fails to infer the X exception type parameter.
-        new AlwaysRetryable<Object>() {
+        new DefaultRetryable<Object>() {
             @Override
             public Object call() throws IOException
             {
@@ -126,6 +126,6 @@ public class TestAlwaysRetryable
     @Test(expected = IllegalStateException.class)
     public void execute_without_an_implementation_should_throw_an_IllegalStateException()
     {
-        new AlwaysRetryable().executeWith(retryExecutor());
+        new DefaultRetryable().executeWith(retryExecutor());
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.19
+  version: 0.2.21
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-26 00:00:00.000000000 Z
+date: 2018-10-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -48,34 +48,34 @@ files:
 - build.gradle
 - lib/embulk/input/s3.rb
 - src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
-- src/main/java/org/embulk/input/s3/AlwaysRetryable.java
+- src/main/java/org/embulk/input/s3/DefaultRetryable.java
 - src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/HttpProxy.java
 - src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java
-- src/test/java/org/embulk/input/s3/TestAlwaysRetryable.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
+- src/test/java/org/embulk/input/s3/TestDefaultRetryable.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestHttpProxy.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
 - src/test/resources/sample_01.csv
-- classpath/aws-java-sdk-core-1.11.253.jar
 - classpath/aws-java-sdk-kms-1.11.253.jar
+- classpath/embulk-input-s3-0.2.21.jar
+- classpath/jackson-core-2.6.7.jar
+- classpath/ion-java-1.0.2.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
-- classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.19.jar
-- classpath/embulk-util-aws-credentials-0.2.19.jar
 - classpath/httpclient-4.5.2.jar
+- classpath/commons-codec-1.9.jar
+- classpath/jmespath-java-1.11.253.jar
+- classpath/jcl-over-slf4j-1.7.12.jar
+- classpath/aws-java-sdk-core-1.11.253.jar
+- classpath/embulk-util-aws-credentials-0.2.21.jar
+- classpath/jackson-dataformat-cbor-2.6.7.jar
+- classpath/jackson-databind-2.6.7.1.jar
 - classpath/httpcore-4.4.4.jar
-- classpath/ion-java-1.0.2.jar
 - classpath/jackson-annotations-2.6.0.jar
-- classpath/jackson-core-2.6.7.jar
-- classpath/jackson-databind-2.6.7.1.jar
-- classpath/jackson-dataformat-cbor-2.6.7.jar
-- classpath/jcl-over-slf4j-1.7.12.jar
-- classpath/jmespath-java-1.11.253.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:
 - Apache 2.0