embulk-input-s3 0.2.14 → 0.2.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a69ebc7bdba28093d994a2e19842c108f265b1a0
-  data.tar.gz: 2ca38204ff6fc9a01d9032e7c6ceb0a3a38f4629
+  metadata.gz: 6efa3b427ccee34a13fae779d053ea5863a676e7
+  data.tar.gz: a54eec2a80f7003d4a7bdeff529e152fe8a7ed3b
 SHA512:
-  metadata.gz: ddc5c1c7f780bd1bc4cad5c1647795d7e25d1564d6d9611970ae86b519f3bb94546e06c6138bc4509df9173453bbeedb2bb31ea90b0759b7dada73b54a1050b9
-  data.tar.gz: d598b8687a1490cfb40d01ea3f730736ae4053848a6ffa630757aad63feab774caa3910e276819db52e58ed72b93a4eedfbe9f56cef7089d925787d77370f987
+  metadata.gz: 35d00e37563f72c184f1bf9f02186927975a14d9eae0bf2ea240fe31c94b28414b70c1574494be19833305a9908e72081a1b5356da570b48d85106366666cdd3
+  data.tar.gz: 7de346f325126fbc1d1ebd5ca7f36269945bc5d7e9227d4b6bfb88f7b0c56327ed2c95b902377b5c84e297898b9e98a6f14021b0634a0da8c18aed23f8d96c63

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -47,7 +47,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 public abstract class AbstractS3FileInputPlugin
         implements FileInputPlugin
 {
-    private final Logger log = Exec.getLogger(S3FileInputPlugin.class);
+    private static final Logger LOGGER = Exec.getLogger(S3FileInputPlugin.class);
 
     public interface PluginTask
             extends AwsCredentialsTask, FileList.Task, Task
@@ -119,9 +119,10 @@ public abstract class AbstractS3FileInputPlugin
 
         // last_path
         if (task.getIncremental()) {
-            configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
+            Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
+            LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orNull());
+            configDiff.set("last_path", lastPath);
         }
-
         return configDiff;
     }
 
@@ -201,17 +202,19 @@ public abstract class AbstractS3FileInputPlugin
 
     private FileList listFiles(PluginTask task)
     {
+        LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix());
         try {
             AmazonS3 client = newS3Client(task);
             String bucketName = task.getBucket();
 
             if (task.getPathPrefix().equals("/")) {
-                log.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
+                LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
             }
 
             FileList.Builder builder = new FileList.Builder(task);
             listS3FilesByPrefix(builder, client, bucketName,
                     task.getPathPrefix(), task.getLastPath(), task.getSkipGlacierObjects());
+            LOGGER.info("Found total [{}] files", builder.size());
             return builder.build();
         }
         catch (AmazonServiceException ex) {
@@ -253,6 +256,7 @@ public abstract class AbstractS3FileInputPlugin
                 if (s.getSize() > 0) {
                     builder.add(s.getKey(), s.getSize());
                     if (!builder.needsMore()) {
+                        LOGGER.warn("Too many files matched, stop listing file");
                         return;
                     }
                 }
@@ -326,6 +330,7 @@ public abstract class AbstractS3FileInputPlugin
                         public void onGiveup(Exception firstException, Exception lastException)
                                 throws RetryGiveupException
                         {
+                            log.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
                         }
                     });
             }
@@ -384,9 +389,12 @@ public abstract class AbstractS3FileInputPlugin
             if (!iterator.hasNext()) {
                 return null;
             }
-            GetObjectRequest request = new GetObjectRequest(bucket, iterator.next());
+            String key = iterator.next();
+            GetObjectRequest request = new GetObjectRequest(bucket, key);
             S3Object obj = client.getObject(request);
-            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, obj.getObjectMetadata().getContentLength()));
+            long objectSize = obj.getObjectMetadata().getContentLength();
+            LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
+            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, objectSize));
         }
 
         @Override

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.15
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-05 00:00:00.000000000 Z
+date: 2018-05-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,8 +61,8 @@ files:
 - classpath/aws-java-sdk-kms-1.11.253.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.14.jar
-- classpath/embulk-util-aws-credentials-0.2.14.jar
+- classpath/embulk-input-s3-0.2.15.jar
+- classpath/embulk-util-aws-credentials-0.2.15.jar
 - classpath/httpclient-4.5.2.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/ion-java-1.0.2.jar