embulk-input-s3 0.2.11 → 0.2.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 034a5456271657abde4e9dc5aac76f7e05efc7e4
-  data.tar.gz: 74f38ae4046a4292487361a81d74c3be201b5aaa
+  metadata.gz: 99dc2bbe73f61b74e92b3f345c963b73ebb75bd0
+  data.tar.gz: 6c42a7ae99a51dad4d9ec4c20b629a9b235db8b3
 SHA512:
-  metadata.gz: 3cc77fc7cf409ce21e2a7373fc983727953a0d22b3c5676c43f4a4c3182a15de487ec3633d3b7b037386451317b5ea5ee2861ce46ae22aff66e1ef0a462b4f42
-  data.tar.gz: 19ff469baf507e141f91a17161d7bf2b48172e273b87b1ef1abb2464c385dc7ea7ced75e64baf04928295bce218203d5f48d2ecedd718efd7291ba53693c78c0
+  metadata.gz: 8dee445a23d719bb9f7bf1fd9dde6f0d25faa2e38e8eec567e8d7a61518841f933caf8e35ce79a27a70327d7b75ab65eb1cb54327d7d4e664c848ed3fc24b265
+  data.tar.gz: 728534ca59e6c3d334335f821d0834bc04f8fd76c5934adfa62132b638a81b1b6dc12874ee718d8352adabe1de3bf26eee3f1a8b3bcf8d117f74076cdc4a3935

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -1,21 +1,18 @@
 package org.embulk.input.s3;
 
 import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
 import java.util.Iterator;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
 
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.slf4j.Logger;
-import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.ObjectListing;
@@ -44,8 +41,6 @@ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.util.aws.credentials.AwsCredentials;
 import org.embulk.util.aws.credentials.AwsCredentialsTask;
 
-import static com.amazonaws.Protocol.HTTP;
-import static com.amazonaws.Protocol.HTTPS;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public abstract class AbstractS3FileInputPlugin
@@ -133,9 +128,22 @@ public abstract class AbstractS3FileInputPlugin
         // do nothing
     }
 
-    protected AmazonS3Client newS3Client(PluginTask task)
-    {
-        return new AmazonS3Client(getCredentialsProvider(task), getClientConfiguration(task));
+    /**
+     * Provide an overridable default client.
+     * Since this returns an immutable object, it is not for any further customizations by mutating,
+     * e.g., {@link AmazonS3#setEndpoint} will throw a runtime {@link UnsupportedOperationException}
+     * Subclass's customization should be done through {@link AbstractS3FileInputPlugin#defaultS3ClientBuilder}.
+     */
+    protected AmazonS3 newS3Client(PluginTask task) {
+        return defaultS3ClientBuilder(task).build();
+    }
+
+    /** A base builder for the subclasses to then customize. */
+    protected AmazonS3ClientBuilder defaultS3ClientBuilder(PluginTask task) {
+        return AmazonS3ClientBuilder
+                .standard()
+                .withCredentials(getCredentialsProvider(task))
+                .withClientConfiguration(getClientConfiguration(task));
     }
 
     protected AWSCredentialsProvider getCredentialsProvider(PluginTask task)
@@ -187,7 +195,7 @@ public abstract class AbstractS3FileInputPlugin
     private FileList listFiles(PluginTask task)
     {
         try {
-            AmazonS3Client client = newS3Client(task);
+            AmazonS3 client = newS3Client(task);
             String bucketName = task.getBucket();
 
             if (task.getPathPrefix().equals("/")) {
@@ -218,7 +226,7 @@ public abstract class AbstractS3FileInputPlugin
      * The resulting list does not include the file that's size == 0.
      */
     public static void listS3FilesByPrefix(FileList.Builder builder,
-            AmazonS3Client client, String bucketName,
+            AmazonS3 client, String bucketName,
             String prefix, Optional<String> lastPath)
     {
         String lastKey = lastPath.orNull();
@@ -250,11 +258,11 @@ public abstract class AbstractS3FileInputPlugin
     {
         private final Logger log = Exec.getLogger(S3InputStreamReopener.class);
 
-        private final AmazonS3Client client;
+        private final AmazonS3 client;
         private final GetObjectRequest request;
         private final long contentLength;
 
-        public S3InputStreamReopener(AmazonS3Client client, GetObjectRequest request, long contentLength)
+        public S3InputStreamReopener(AmazonS3 client, GetObjectRequest request, long contentLength)
         {
             this.client = client;
             this.request = request;
@@ -336,7 +344,7 @@ public abstract class AbstractS3FileInputPlugin
     private class SingleFileProvider
             implements InputStreamFileInput.Provider
     {
-        private AmazonS3Client client;
+        private AmazonS3 client;
         private final String bucket;
         private final Iterator<String> iterator;
 

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -1,10 +1,16 @@
 package org.embulk.input.s3;
 
-import com.google.common.base.Optional;
+import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
+import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.input.s3.AbstractS3FileInputPlugin;
+
+import static com.amazonaws.services.s3.AmazonS3Client.S3_SERVICE_NAME;
+import static com.amazonaws.util.AwsHostNameUtils.parseRegion;
+import static com.amazonaws.util.RuntimeHttpUtils.toUri;
 
 public class S3FileInputPlugin
         extends AbstractS3FileInputPlugin
@@ -24,16 +30,24 @@ public class S3FileInputPlugin
     }
 
     @Override
-    protected AmazonS3Client newS3Client(PluginTask task)
+    protected AmazonS3 newS3Client(PluginTask task)
     {
         S3PluginTask t = (S3PluginTask) task;
 
-        AmazonS3Client client = super.newS3Client(t);
+        AmazonS3ClientBuilder builder = super.defaultS3ClientBuilder(t);
 
         if (t.getEndpoint().isPresent()) {
-            client.setEndpoint(t.getEndpoint().get());
+            String endpoint = t.getEndpoint().get();
+            builder.setEndpointConfiguration(new EndpointConfiguration(
+                    endpoint,
+                    // Although client will treat endpoint's region as the signer region
+                    // if we left this as null, but such that behaviour is undocumented,
+                    // so it is explicitly calculated here for future-proofing.
+                    parseRegion(
+                            toUri(endpoint, getClientConfiguration(task)).getHost(),
+                            S3_SERVICE_NAME)));
         }
 
-        return client;
+        return builder.build();
     }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.12
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-27 00:00:00.000000000 Z
+date: 2018-01-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,15 +57,21 @@ files:
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
 - src/test/resources/sample_01.csv
-- classpath/aws-java-sdk-core-1.10.33.jar
-- classpath/aws-java-sdk-kms-1.10.33.jar
-- classpath/aws-java-sdk-s3-1.10.33.jar
-- classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.11.jar
-- classpath/embulk-util-aws-credentials-0.2.11.jar
-- classpath/httpclient-4.3.6.jar
-- classpath/httpcore-4.3.3.jar
+- classpath/aws-java-sdk-core-1.11.253.jar
+- classpath/aws-java-sdk-kms-1.11.253.jar
+- classpath/aws-java-sdk-s3-1.11.253.jar
+- classpath/commons-codec-1.9.jar
+- classpath/embulk-input-s3-0.2.12.jar
+- classpath/embulk-util-aws-credentials-0.2.12.jar
+- classpath/httpclient-4.5.2.jar
+- classpath/httpcore-4.4.4.jar
+- classpath/ion-java-1.0.2.jar
+- classpath/jackson-annotations-2.6.0.jar
+- classpath/jackson-core-2.6.7.jar
+- classpath/jackson-databind-2.6.7.1.jar
+- classpath/jackson-dataformat-cbor-2.6.7.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
+- classpath/jmespath-java-1.11.253.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:
 - Apache 2.0