embulk-input-s3 0.3.0 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7aef07b030009a6352551d4b7ebd78be174dbf53
-  data.tar.gz: ec9ecba65e22bc04e73819d145ccb2a6a4d8115e
+  metadata.gz: d77ba197fb47f89fc3a890e240ddd963a5acc9fd
+  data.tar.gz: 97647120fdd11ddc13e03916a11429477c707ba4
 SHA512:
-  metadata.gz: 867c41e167c9addeeabc893781c912880756b9ef5a16554a16440972acd30c425500b927a17736d2892cff167d64c352e11c87386808f756453290beb0b738a7
-  data.tar.gz: 07f0fea1f0716c3b74d384eb8174f6193d84e0714dabfd8fa4bf8cdaaba6b25f27f1a26aa5a1746115a89a095ae3799d3000350ffdf890be4255bed29a45d369
+  metadata.gz: d81311e7e1e921bc336de2a78c01230616104a3c93ca6ba7699fd934c8536588bb0fd813033fd596774a5995641b753eaf6007d489e3f44daf30edbee95df215
+  data.tar.gz: f4d71fe3d8be542bd7563f7b52d2503d8840314f175ab41fecfb2bf27498188a8ec3e0a58828fcc022bb4b650396e1559b87684134d0d232b65dce633c4450dd

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -7,15 +7,9 @@ import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.retry.PredefinedRetryPolicies;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.annotations.VisibleForTesting;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -26,6 +20,10 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
+import org.embulk.input.s3.explorer.S3NameOrderPrefixFileExplorer;
+import org.embulk.input.s3.explorer.S3SingleFileExplorer;
+import org.embulk.input.s3.explorer.S3TimeOrderPrefixFileExplorer;
+import org.embulk.input.s3.utils.DateUtils;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -40,6 +38,9 @@ import org.slf4j.Logger;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.text.SimpleDateFormat;
+import java.util.Collections;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Optional;
@@ -51,6 +52,7 @@ public abstract class AbstractS3FileInputPlugin
         implements FileInputPlugin
 {
     private static final Logger LOGGER = Exec.getLogger(S3FileInputPlugin.class);
+    private static final String FULL_DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
 
     public interface PluginTask
             extends AwsCredentialsTask, FileList.Task, RetrySupportPluginTask, Task
@@ -88,12 +90,35 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("false")
         boolean getSkipGlacierObjects();
 
+        @Config("use_modified_time")
+        @ConfigDefault("false")
+        boolean getUseModifiedTime();
+
+        @Config("last_modified_time")
+        @ConfigDefault("null")
+        Optional<String> getLastModifiedTime();
+
         // TODO timeout, ssl, etc
 
+        ////////////////////////////////////////
+        // Internal configurations
+        ////////////////////////////////////////
+
         FileList getFiles();
 
         void setFiles(FileList files);
 
+        /**
+         * end_modified_time is conditionally set if modified_time mode is enabled.
+         *
+         * It is internal state and must not be set in config.yml
+         */
+        @Config("__end_modified_time")
+        @ConfigDefault("null")
+        Optional<Date> getEndModifiedTime();
+
+        void setEndModifiedTime(Optional<Date> endModifiedTime);
+
         @ConfigInject
         BufferAllocator getBufferAllocator();
     }
@@ -105,6 +130,7 @@ public abstract class AbstractS3FileInputPlugin
     {
         PluginTask task = config.loadConfig(getTaskClass());
 
+        errorIfInternalParamsAreSet(task);
         validateInputTask(task);
         // list files recursively
         task.setFiles(listFiles(task));
@@ -130,9 +156,15 @@ public abstract class AbstractS3FileInputPlugin
 
         // last_path
         if (task.getIncremental()) {
-            Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
-            LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
-            configDiff.set("last_path", lastPath);
+            if (task.getUseModifiedTime()) {
+                Date endModifiedTime = task.getEndModifiedTime().orElse(new Date());
+                configDiff.set("last_modified_time", new SimpleDateFormat(FULL_DATE_FORMAT).format(endModifiedTime));
+            }
+            else {
+                Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
+                LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
+                configDiff.set("last_path", lastPath);
+            }
         }
         return configDiff;
     }
@@ -180,11 +212,10 @@ public abstract class AbstractS3FileInputPlugin
     {
         ClientConfiguration clientConfig = new ClientConfiguration();
 
-        /** PLT-9886: disable built-in retry*/
         //clientConfig.setProtocol(Protocol.HTTP);
-//        clientConfig.setMaxConnections(50); // SDK default: 50
+        clientConfig.setMaxConnections(50); // SDK default: 50
 //        clientConfig.setMaxErrorRetry(3); // SDK default: 3
-//        clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
+        clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
         clientConfig.setRetryPolicy(PredefinedRetryPolicies.NO_RETRY_POLICY);
         // set http proxy
         if (task.getHttpProxy().isPresent()) {
@@ -238,22 +269,35 @@ public abstract class AbstractS3FileInputPlugin
             String bucketName = task.getBucket();
             FileList.Builder builder = new FileList.Builder(task);
             RetryExecutor retryExec = retryExecutorFrom(task);
+
             if (task.getPath().isPresent()) {
                 LOGGER.info("Start getting object with path: [{}]", task.getPath().get());
-                addS3DirectObject(builder, client, task.getBucket(), task.getPath().get(), retryExec);
+                new S3SingleFileExplorer(bucketName, client, retryExec, task.getPath().get()).addToBuilder(builder);
+                return builder.build();
             }
-            else {
-                // does not need to verify existent path prefix here since there is the validation requires either path or path_prefix
-                LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix().get());
-                if (task.getPathPrefix().get().equals("/")) {
-                    LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
-                }
 
-                listS3FilesByPrefix(builder, client, bucketName,
-                        task.getPathPrefix().get(), task.getLastPath(), task.getSkipGlacierObjects(), retryExec);
-                LOGGER.info("Found total [{}] files", builder.size());
+            // does not need to verify existent path prefix here since there is the validation requires either path or path_prefix
+            LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix().get());
+            if (task.getPathPrefix().get().equals("/")) {
+                LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
             }
 
+            if (task.getUseModifiedTime()) {
+                Date now = new Date();
+                Optional<Date> from = task.getLastModifiedTime().isPresent()
+                        ? Optional.of(DateUtils.parse(task.getLastModifiedTime().get(), Collections.singletonList(FULL_DATE_FORMAT)))
+                        : Optional.empty();
+                task.setEndModifiedTime(Optional.of(now));
+
+                new S3TimeOrderPrefixFileExplorer(bucketName, client, retryExec, task.getPathPrefix().get(),
+                        task.getSkipGlacierObjects(), from, now).addToBuilder(builder);
+            }
+            else {
+                new S3NameOrderPrefixFileExplorer(bucketName, client, retryExec, task.getPathPrefix().get(),
+                        task.getSkipGlacierObjects(), task.getLastPath().orElse(null)).addToBuilder(builder);
+            }
+
+            LOGGER.info("Found total [{}] files", builder.size());
             return builder.build();
         }
         catch (AmazonServiceException ex) {
@@ -269,107 +313,13 @@ public abstract class AbstractS3FileInputPlugin
         }
     }
 
-    @VisibleForTesting
-    public void addS3DirectObject(FileList.Builder builder,
-                                  final AmazonS3 client,
-                                  String bucket,
-                                  String objectKey)
-    {
-        addS3DirectObject(builder, client, bucket, objectKey, null);
-    }
-
-    @VisibleForTesting
-    public void addS3DirectObject(FileList.Builder builder,
-                                   final AmazonS3 client,
-                                   String bucket,
-                                   String objectKey,
-                                   RetryExecutor retryExec)
-    {
-        final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey);
-
-        ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
-            @Override
-            public ObjectMetadata call()
-            {
-                return client.getObjectMetadata(objectMetadataRequest);
-            }
-        }.executeWith(retryExec);
-
-        builder.add(objectKey, objectMetadata.getContentLength());
-    }
-
-    private void validateInputTask(PluginTask task)
+    private void validateInputTask(final PluginTask task)
     {
         if (!task.getPathPrefix().isPresent() && !task.getPath().isPresent()) {
             throw new ConfigException("Either path or path_prefix is required");
         }
     }
 
-    @VisibleForTesting
-    public static void listS3FilesByPrefix(FileList.Builder builder,
-                                           final AmazonS3 client,
-                                           String bucketName,
-                                           String prefix,
-                                           Optional<String> lastPath,
-                                           boolean skipGlacierObjects)
-    {
-        listS3FilesByPrefix(builder, client, bucketName, prefix, lastPath, skipGlacierObjects, null);
-    }
-
-    /**
-     * Lists S3 filenames filtered by prefix.
-     * <p>
-     * The resulting list does not include the file that's size == 0.
-     * @param builder custom Filelist builder
-     * @param client Amazon S3
-     * @param bucketName Amazon S3 bucket name
-     * @param prefix Amazon S3 bucket name prefix
-     * @param lastPath last path
-     * @param skipGlacierObjects skip gracier objects
-     * @param retryExec a retry executor object to do the retrying
-     */
-    @VisibleForTesting
-    public static void listS3FilesByPrefix(FileList.Builder builder,
-                                           final AmazonS3 client,
-                                           String bucketName,
-                                           String prefix,
-                                           Optional<String> lastPath,
-                                           boolean skipGlacierObjects,
-                                           RetryExecutor retryExec)
-    {
-        String lastKey = lastPath.orElse(null);
-        do {
-            final String finalLastKey = lastKey;
-            final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
-            ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects") {
-                @Override
-                public ObjectListing call()
-                {
-                    return client.listObjects(req);
-                }
-            }.executeWith(retryExec);
-            for (S3ObjectSummary s : ol.getObjectSummaries()) {
-                if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
-                    if (skipGlacierObjects) {
-                        Exec.getLogger("AbstractS3FileInputPlugin.class").warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
-                        continue;
-                    }
-                    else {
-                        throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
-                    }
-                }
-                if (s.getSize() > 0) {
-                    builder.add(s.getKey(), s.getSize());
-                    if (!builder.needsMore()) {
-                        LOGGER.warn("Too many files matched, stop listing file");
-                        return;
-                    }
-                }
-            }
-            lastKey = ol.getNextMarker();
-        } while (lastKey != null);
-    }
-
     @Override
     public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
     {
@@ -441,6 +391,14 @@ public abstract class AbstractS3FileInputPlugin
         }
     }
 
+    @VisibleForTesting
+    static void errorIfInternalParamsAreSet(PluginTask task)
+    {
+        if (task.getEndModifiedTime().isPresent()) {
+            throw new ConfigException("'__end_modified_time' must not be set.");
+        }
+    }
+
     // TODO create single-file InputStreamFileInput utility
     private class SingleFileProvider
             implements InputStreamFileInput.Provider
@@ -476,6 +434,9 @@ public abstract class AbstractS3FileInputPlugin
             }.executeWithCheckedException(retryExec, IOException.class);
 
             long objectSize = object.getObjectMetadata().getContentLength();
+            // Some plugin users are parsing this output to get file list.
+            // Keep it for now but might be removed in the future.
+            LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
             InputStream inputStream = new ResumableInputStream(object.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec));
             return new InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key));
         }

data/src/main/java/org/embulk/input/s3/DefaultRetryable.java

@@ -19,7 +19,7 @@ import static org.embulk.spi.util.RetryExecutor.Retryable;
  * Retryable utility, regardless the occurred exceptions,
  * Also provide a default approach for exception propagation.
  */
-class DefaultRetryable<T> implements Retryable<T>
+public class DefaultRetryable<T> implements Retryable<T>
 {
     private static final Logger log = Exec.getLogger(DefaultRetryable.class);
     private static final Set<Integer> NONRETRYABLE_STATUS_CODES = new HashSet<Integer>(2);

data/src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java

@@ -11,7 +11,7 @@ public interface RetrySupportPluginTask extends Task
     int getMaximumRetries();
 
     @Config("initial_retry_interval_millis")
-    @ConfigDefault("30000")
+    @ConfigDefault("2000")
     int getInitialRetryIntervalMillis();
 
     @Config("maximum_retry_interval_millis")

data/src/main/java/org/embulk/input/s3/explorer/S3FileExplorer.java

@@ -0,0 +1,21 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+
+public abstract class S3FileExplorer
+{
+    protected String bucketName;
+    protected AmazonS3 s3Client;
+    protected RetryExecutor retryExecutor;
+
+    public S3FileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor)
+    {
+        this.bucketName = bucketName;
+        this.s3Client = s3Client;
+        this.retryExecutor = retryExecutor;
+    }
+
+    public abstract void addToBuilder(FileList.Builder builder);
+}

data/src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java

@@ -0,0 +1,45 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.spi.util.RetryExecutor;
+
+import java.util.List;
+
+public class S3NameOrderPrefixFileExplorer extends S3PrefixFileExplorer
+{
+    private String lastPath;
+
+    public S3NameOrderPrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor,
+            final String pathPrefix, final boolean skipGlacierObjects, final String lastPath)
+    {
+        super(bucketName, s3Client, retryExecutor, pathPrefix, skipGlacierObjects);
+        this.lastPath = lastPath;
+    }
+
+    @Override
+    protected List<S3ObjectSummary> fetch()
+    {
+        final ListObjectsRequest req = new ListObjectsRequest(bucketName, pathPrefix, lastPath, null, 1024);
+        final ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects")
+        {
+            @Override
+            public ObjectListing call()
+            {
+                return s3Client.listObjects(req);
+            }
+        }.executeWith(retryExecutor);
+        lastPath = ol.getNextMarker();
+
+        return ol.getObjectSummaries();
+    }
+
+    @Override
+    protected boolean hasNext()
+    {
+        return lastPath != null;
+    }
+}

data/src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java

@@ -0,0 +1,57 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
+import org.embulk.config.ConfigException;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.util.List;
+
+public abstract class S3PrefixFileExplorer extends S3FileExplorer
+{
+    private static final Logger LOGGER = Exec.getLogger(S3PrefixFileExplorer.class);
+
+    protected String pathPrefix;
+
+    private final boolean skipGlacierObjects;
+
+    public S3PrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor, final String pathPrefix, final boolean skipGlacierObjects)
+    {
+        super(bucketName, s3Client, retryExecutor);
+        this.pathPrefix = pathPrefix;
+        this.skipGlacierObjects = skipGlacierObjects;
+    }
+
+    @Override
+    public void addToBuilder(final FileList.Builder builder)
+    {
+        do {
+            final List<S3ObjectSummary> s3ObjectSummaries = fetch();
+
+            for (final S3ObjectSummary s : s3ObjectSummaries) {
+                if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
+                    if (skipGlacierObjects) {
+                        LOGGER.warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
+                        continue;
+                    }
+                    throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
+                }
+                if (s.getSize() > 0) {
+                    builder.add(s.getKey(), s.getSize());
+                    if (!builder.needsMore()) {
+                        LOGGER.warn("Too many files matched, stop listing file");
+                        return;
+                    }
+                }
+            }
+        } while (hasNext());
+    }
+
+    protected abstract List<S3ObjectSummary> fetch();
+
+    protected abstract boolean hasNext();
+}