embulk-input-s3 0.2.12 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99dc2bbe73f61b74e92b3f345c963b73ebb75bd0
-  data.tar.gz: 6c42a7ae99a51dad4d9ec4c20b629a9b235db8b3
+  metadata.gz: f74f5e05b21a26a23ae93264b7fb5a6624833eda
+  data.tar.gz: 9dd80d007fb72660b3670845a6530a2ca75f316f
 SHA512:
-  metadata.gz: 8dee445a23d719bb9f7bf1fd9dde6f0d25faa2e38e8eec567e8d7a61518841f933caf8e35ce79a27a70327d7b75ab65eb1cb54327d7d4e664c848ed3fc24b265
-  data.tar.gz: 728534ca59e6c3d334335f821d0834bc04f8fd76c5934adfa62132b638a81b1b6dc12874ee718d8352adabe1de3bf26eee3f1a8b3bcf8d117f74076cdc4a3935
+  metadata.gz: d93ad5992045608cb56d6a4753119b6dbe6c84c71ce0a360a2cc2001a73f9cb694c4250045e96ccb119cc782278adfaa8087f4b412714fb9ad8a56bcca18774d
+  data.tar.gz: db5c37847557095a02e86b3d0d75ad97e13b1e26744d055d38e688e527166c505d3075f5b65b8f16e9d2290bce25800fd8f316509b70710dab4d2207431a1e2f

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -1,45 +1,45 @@
 package org.embulk.input.s3;
 
-import java.util.List;
-import java.util.Iterator;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.InputStream;
-
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.slf4j.Logger;
-import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.Protocol;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
 import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.TaskReport;
 import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.ResumableInputStream;
-import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.util.Iterator;
+import java.util.List;
 
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
@@ -134,12 +134,14 @@ public abstract class AbstractS3FileInputPlugin
      * e.g., {@link AmazonS3#setEndpoint} will throw a runtime {@link UnsupportedOperationException}
      * Subclass's customization should be done through {@link AbstractS3FileInputPlugin#defaultS3ClientBuilder}.
      */
-    protected AmazonS3 newS3Client(PluginTask task) {
+    protected AmazonS3 newS3Client(PluginTask task)
+    {
         return defaultS3ClientBuilder(task).build();
     }
 
     /** A base builder for the subclasses to then customize. */
-    protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task) {
+    protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task)
+    {
         return AmazonS3ClientBuilder
                 .standard()
                 .withCredentials(getCredentialsProvider(task))
@@ -158,7 +160,7 @@ public abstract class AbstractS3FileInputPlugin
         //clientConfig.setProtocol(Protocol.HTTP);
         clientConfig.setMaxConnections(50); // SDK default: 50
         clientConfig.setMaxErrorRetry(3); // SDK default: 3
-        clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
+        clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
 
         // set http proxy
         if (task.getHttpProxy().isPresent()) {
@@ -276,7 +278,7 @@ public abstract class AbstractS3FileInputPlugin
                 return retryExecutor()
                     .withRetryLimit(3)
                     .withInitialRetryWait(500)
-                    .withMaxRetryWait(30*1000)
+                    .withMaxRetryWait(30 * 1000)
                     .runInterruptible(new Retryable<InputStream>() {
                         @Override
                         public InputStream call() throws InterruptedIOException
@@ -297,10 +299,11 @@ public abstract class AbstractS3FileInputPlugin
                                 throws RetryGiveupException
                         {
                             String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
                             if (retryCount % 3 == 0) {
                                 log.warn(message, exception);
-                            } else {
+                            }
+                            else {
                                 log.warn(message);
                             }
                         }
@@ -311,10 +314,12 @@ public abstract class AbstractS3FileInputPlugin
                         {
                         }
                     });
-            } catch (RetryGiveupException ex) {
+            }
+            catch (RetryGiveupException ex) {
                 Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
                 throw Throwables.propagate(ex.getCause());
-            } catch (InterruptedException ex) {
+            }
+            catch (InterruptedException ex) {
                 throw new InterruptedIOException();
             }
         }
@@ -329,7 +334,9 @@ public abstract class AbstractS3FileInputPlugin
             super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
         }
 
-        public void abort() { }
+        public void abort()
+        {
+        }
 
         public TaskReport commit()
         {
@@ -337,7 +344,9 @@ public abstract class AbstractS3FileInputPlugin
         }
 
         @Override
-        public void close() { }
+        public void close()
+        {
+        }
     }
 
     // TODO create single-file InputStreamFileInput utility
@@ -367,6 +376,8 @@ public abstract class AbstractS3FileInputPlugin
         }
 
         @Override
-        public void close() { }
+        public void close()
+        {
+        }
     }
 }

data/src/main/java/org/embulk/input/s3/FileList.java

@@ -1,30 +1,31 @@
 package org.embulk.input.s3;
 
-import java.util.List;
-import java.util.AbstractList;
-import java.util.ArrayList;
-import java.util.zip.GZIPInputStream;
-import java.util.zip.GZIPOutputStream;
-import java.util.regex.Pattern;
-import java.util.regex.Matcher;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.BufferedOutputStream;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+
 import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
+
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
 import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigSource;
-import com.google.common.base.Throwables;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.fasterxml.jackson.annotation.JsonIgnore;
-import com.fasterxml.jackson.annotation.JsonCreator;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 
 // this class should be moved to embulk-core
 public class FileList
@@ -60,10 +61,16 @@ public class FileList
         }
 
         @JsonProperty("index")
-        public int getIndex() { return index; }
+        public int getIndex()
+        {
+            return index;
+        }
 
         @JsonProperty("size")
-        public long getSize() { return size; }
+        public long getSize()
+        {
+            return size;
+        }
     }
 
     public static class Builder

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -2,15 +2,12 @@ package org.embulk.input.s3;
 
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-
-import static com.amazonaws.services.s3.AmazonS3Client.S3_SERVICE_NAME;
-import static com.amazonaws.util.AwsHostNameUtils.parseRegion;
-import static com.amazonaws.util.RuntimeHttpUtils.toUri;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
 
 public class S3FileInputPlugin
         extends AbstractS3FileInputPlugin
@@ -21,11 +18,16 @@ public class S3FileInputPlugin
         @Config("endpoint")
         @ConfigDefault("null")
         public Optional<String> getEndpoint();
+
+        @Config("region")
+        @ConfigDefault("null")
+        public Optional<String> getRegion();
     }
 
+    private static final Logger log = Exec.getLogger(S3FileInputPlugin.class);
+
     @Override
-    protected Class<? extends PluginTask> getTaskClass()
-    {
+    protected Class<? extends PluginTask> getTaskClass() {
         return S3PluginTask.class;
     }
 
@@ -33,19 +35,35 @@ public class S3FileInputPlugin
     protected AmazonS3 newS3Client(PluginTask task)
     {
         S3PluginTask t = (S3PluginTask) task;
+        Optional<String> endpoint = t.getEndpoint();
+        Optional<String> region = t.getRegion();
 
         AmazonS3ClientBuilder builder = super.defaultS3ClientBuilder(t);
 
-        if (t.getEndpoint().isPresent()) {
-            String endpoint = t.getEndpoint().get();
-            builder.setEndpointConfiguration(new EndpointConfiguration(
-                    endpoint,
-                    // Although client will treat endpoint's region as the signer region
-                    // if we left this as null, but such that behaviour is undocumented,
-                    // so it is explicitly calculated here for future-proofing.
-                    parseRegion(
-                            toUri(endpoint, getClientConfiguration(task)).getHost(),
-                            S3_SERVICE_NAME)));
+        // Favor the `endpoint` configuration, then `region`, if both are absent then `s3.amazonaws.com` will be used.
+        if (endpoint.isPresent()) {
+            if (region.isPresent()) {
+                log.warn("Either configure endpoint or region, " +
+                        "if both is specified only the endpoint will be in effect.");
+            }
+            builder.setEndpointConfiguration(new EndpointConfiguration(endpoint.get(), null));
+        } else if (region.isPresent()) {
+            builder.setRegion(region.get());
+        } else {
+            // This is to keep the AWS SDK upgrading to 1.11.x to be backward compatible with old configuration.
+            //
+            // On SDK 1.10.x, when neither endpoint nor region is set explicitly, the client's endpoint will be by
+            // default `s3.amazonaws.com`. And for pre-Signature-V4, this will work fine as the bucket's region
+            // will be resolved to the appropriate region on server (AWS) side.
+            //
+            // On SDK 1.11.x, a region will be computed on client side by AwsRegionProvider and the endpoint now will
+            // be region-specific `<region>.s3.amazonaws.com` and might be the wrong one.
+            //
+            // So a default endpoint of `s3.amazonaws.com` when both endpoint and region configs are absent are
+            // necessary to make old configurations won't suddenly break. The side effect is that this will render
+            // AwsRegionProvider useless. And it's worth to note that Signature-V4 won't work with either versions with
+            // no explicit region or endpoint as the region (inferrable from endpoint) are necessary for signing.
+            builder.setEndpointConfiguration(new EndpointConfiguration("s3.amazonaws.com", null));
         }
 
         return builder.build();

data/src/test/java/org/embulk/input/s3/TestAwsCredentials.java

@@ -1,13 +1,14 @@
 package org.embulk.input.s3;
 
+import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.auth.policy.Policy;
 import com.amazonaws.auth.policy.Resource;
 import com.amazonaws.auth.policy.Statement;
 import com.amazonaws.auth.policy.actions.S3Actions;
-import com.amazonaws.internal.StaticCredentialsProvider;
-import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenService;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClientBuilder;
 import com.amazonaws.services.securitytoken.model.Credentials;
 import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
 import com.amazonaws.services.securitytoken.model.GetFederationTokenResult;
@@ -112,7 +113,6 @@ public class TestAwsCredentials
         String origAccessKeyId = System.getProperty("aws.accessKeyId");
         String origSecretKey = System.getProperty("aws.secretKey");
         try {
-
             ConfigSource config = this.config.deepCopy().set("auth_method", "properties");
             System.setProperty("aws.accessKeyId", EMBULK_S3_TEST_ACCESS_KEY_ID);
             System.setProperty("aws.secretKey", EMBULK_S3_TEST_SECRET_ACCESS_KEY);
@@ -148,8 +148,9 @@ public class TestAwsCredentials
 
     private static BasicSessionCredentials getSessionCredentials()
     {
-        AWSSecurityTokenServiceClient stsClient = new AWSSecurityTokenServiceClient(
-                new StaticCredentialsProvider(new BasicAWSCredentials(EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY)));
+        AWSSecurityTokenService stsClient = AWSSecurityTokenServiceClientBuilder.standard().withCredentials(
+                new AWSStaticCredentialsProvider(new BasicAWSCredentials(EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY))
+        ).build();
 
         GetFederationTokenRequest getFederationTokenRequest = new GetFederationTokenRequest();
         getFederationTokenRequest.setDurationSeconds(7200);

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -1,5 +1,7 @@
 package org.embulk.input.s3;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.Region;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
@@ -22,6 +24,7 @@ import org.junit.Test;
 import java.util.ArrayList;
 import java.util.List;
 
+import static org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
@@ -147,6 +150,51 @@ public class TestS3FileInputPlugin
         }
     }
 
+
+    @Test
+    public void configuredEndpoint()
+    {
+        S3PluginTask task = config.deepCopy()
+                .set("endpoint", "s3-ap-southeast-1.amazonaws.com")
+                .set("region", "ap-southeast-2")
+                .loadConfig(S3PluginTask.class);
+        S3FileInputPlugin plugin = runtime.getInstance(S3FileInputPlugin.class);
+        AmazonS3 s3Client = plugin.newS3Client(task);
+
+        // Should not crash and favor the endpoint over the region configuration (there's a warning log though)
+        assertEquals(s3Client.getRegion(), Region.AP_Singapore);
+    }
+
+    @Test
+    public void configuredRegion()
+    {
+        S3PluginTask task = config.deepCopy()
+                .set("region", "ap-southeast-2")
+                .remove("endpoint")
+                .loadConfig(S3PluginTask.class);
+        S3FileInputPlugin plugin = runtime.getInstance(S3FileInputPlugin.class);
+        AmazonS3 s3Client = plugin.newS3Client(task);
+
+        // Should reflect the region configuration as is
+        assertEquals(s3Client.getRegion(), Region.AP_Sydney);
+    }
+
+    @Test
+    public void unconfiguredEndpointAndRegion()
+    {
+        S3PluginTask task = config.deepCopy()
+                .remove("endpoint")
+                .remove("region")
+                .loadConfig(S3PluginTask.class);
+        S3FileInputPlugin plugin = runtime.getInstance(S3FileInputPlugin.class);
+        AmazonS3 s3Client = plugin.newS3Client(task);
+
+        // US Standard region is a 'generic' one (s3.amazonaws.com), the expectation here that
+        // the S3 client should not eagerly resolves for a specific region on client side.
+        // Please refer to org.embulk.input.s3.S3FileInputPlugin#newS3Client for the details.
+        assertEquals(s3Client.getRegion(), Region.US_Standard);
+    }
+
     static class Control
             implements InputPlugin.Control
     {

data/src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java

@@ -1,6 +1,6 @@
 package org.embulk.input.s3;
 
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
@@ -27,12 +27,12 @@ public class TestS3InputStreamReopener
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
-    private AmazonS3Client client;
+    private AmazonS3 client;
 
     @Before
     public void createResources()
     {
-        client = mock(AmazonS3Client.class);
+        client = mock(AmazonS3.class);
     }
 
     @Test

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.12
+  version: 0.2.13
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-01-05 00:00:00.000000000 Z
+date: 2018-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,8 +61,8 @@ files:
 - classpath/aws-java-sdk-kms-1.11.253.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.12.jar
-- classpath/embulk-util-aws-credentials-0.2.12.jar
+- classpath/embulk-input-s3-0.2.13.jar
+- classpath/embulk-util-aws-credentials-0.2.13.jar
 - classpath/httpclient-4.5.2.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/ion-java-1.0.2.jar