embulk-input-s3 0.2.13 → 0.2.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f74f5e05b21a26a23ae93264b7fb5a6624833eda
-  data.tar.gz: 9dd80d007fb72660b3670845a6530a2ca75f316f
+  metadata.gz: a69ebc7bdba28093d994a2e19842c108f265b1a0
+  data.tar.gz: 2ca38204ff6fc9a01d9032e7c6ceb0a3a38f4629
 SHA512:
-  metadata.gz: d93ad5992045608cb56d6a4753119b6dbe6c84c71ce0a360a2cc2001a73f9cb694c4250045e96ccb119cc782278adfaa8087f4b412714fb9ad8a56bcca18774d
-  data.tar.gz: db5c37847557095a02e86b3d0d75ad97e13b1e26744d055d38e688e527166c505d3075f5b65b8f16e9d2290bce25800fd8f316509b70710dab4d2207431a1e2f
+  metadata.gz: ddc5c1c7f780bd1bc4cad5c1647795d7e25d1564d6d9611970ae86b519f3bb94546e06c6138bc4509df9173453bbeedb2bb31ea90b0759b7dada73b54a1050b9
+  data.tar.gz: d598b8687a1490cfb40d01ea3f730736ae4053848a6ffa630757aad63feab774caa3910e276819db52e58ed72b93a4eedfbe9f56cef7089d925787d77370f987

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -11,6 +11,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
@@ -74,6 +75,10 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("true")
         public boolean getIncremental();
 
+        @Config("skip_glacier_objects")
+        @ConfigDefault("false")
+        public boolean getSkipGlacierObjects();
+
         // TODO timeout, ssl, etc
 
         public FileList getFiles();
@@ -206,7 +211,7 @@ public abstract class AbstractS3FileInputPlugin
 
             FileList.Builder builder = new FileList.Builder(task);
             listS3FilesByPrefix(builder, client, bucketName,
-                    task.getPathPrefix(), task.getLastPath());
+                    task.getPathPrefix(), task.getLastPath(), task.getSkipGlacierObjects());
             return builder.build();
         }
         catch (AmazonServiceException ex) {
@@ -229,13 +234,22 @@ public abstract class AbstractS3FileInputPlugin
      */
     public static void listS3FilesByPrefix(FileList.Builder builder,
             AmazonS3 client, String bucketName,
-            String prefix, Optional<String> lastPath)
+            String prefix, Optional<String> lastPath, boolean skipGlacierObjects)
     {
         String lastKey = lastPath.orNull();
         do {
             ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
             ObjectListing ol = client.listObjects(req);
             for (S3ObjectSummary s : ol.getObjectSummaries()) {
+                if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
+                    if (skipGlacierObjects) {
+                        Exec.getLogger("AbstractS3FileInputPlugin.class").warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
+                        continue;
+                    }
+                    else {
+                        throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
+                    }
+                }
                 if (s.getSize() > 0) {
                     builder.add(s.getKey(), s.getSize());
                     if (!builder.needsMore()) {

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -27,7 +27,8 @@ public class S3FileInputPlugin
     private static final Logger log = Exec.getLogger(S3FileInputPlugin.class);
 
     @Override
-    protected Class<? extends PluginTask> getTaskClass() {
+    protected Class<? extends PluginTask> getTaskClass()
+    {
         return S3PluginTask.class;
     }
 
@@ -47,9 +48,11 @@ public class S3FileInputPlugin
                         "if both is specified only the endpoint will be in effect.");
             }
             builder.setEndpointConfiguration(new EndpointConfiguration(endpoint.get(), null));
-        } else if (region.isPresent()) {
+        }
+        else if (region.isPresent()) {
             builder.setRegion(region.get());
-        } else {
+        }
+        else {
             // This is to keep the AWS SDK upgrading to 1.11.x to be backward compatible with old configuration.
             //
             // On SDK 1.10.x, when neither endpoint nor region is set explicitly, the client's endpoint will be by

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -1,11 +1,17 @@
 package org.embulk.input.s3;
 
 import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.Region;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
+import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -20,7 +26,9 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
+import org.mockito.Mockito;
 
+import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
 
@@ -29,6 +37,9 @@ import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assume.assumeNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
 
 public class TestS3FileInputPlugin
 {
@@ -150,7 +161,6 @@ public class TestS3FileInputPlugin
         }
     }
 
-
     @Test
     public void configuredEndpoint()
     {
@@ -195,6 +205,44 @@ public class TestS3FileInputPlugin
         assertEquals(s3Client.getRegion(), Region.US_Standard);
     }
 
+    @Test(expected = ConfigException.class)
+    public void useSkipGlacierObjects() throws Exception
+    {
+        AmazonS3 client;
+        client = mock(AmazonS3.class);
+        doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class));
+
+        AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS);
+        plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.<String>absent(), false);
+    }
+
+    private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize)
+    {
+        FileList.Builder builder = new FileList.Builder(config);
+        for (int i = 0; i < nameAndSize.length; i += 2) {
+            builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
+        }
+        return builder;
+    }
+
+    private ObjectListing s3objectList(String key, StorageClass storageClass) throws Exception
+    {
+        ObjectListing list = new ObjectListing();
+
+        S3ObjectSummary element = new S3ObjectSummary();
+        element.setKey(key);
+        element.setStorageClass(storageClass.toString());
+
+        List<S3ObjectSummary> objectSummaries = new ArrayList<>();
+        objectSummaries.add(element);
+
+        Field field = list.getClass().getDeclaredField("objectSummaries");
+        field.setAccessible(true);
+        field.set(list, objectSummaries);
+
+        return list;
+    }
+
     static class Control
             implements InputPlugin.Control
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.13
+  version: 0.2.14
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-01 00:00:00.000000000 Z
+date: 2018-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,8 +61,8 @@ files:
 - classpath/aws-java-sdk-kms-1.11.253.jar
 - classpath/aws-java-sdk-s3-1.11.253.jar
 - classpath/commons-codec-1.9.jar
-- classpath/embulk-input-s3-0.2.13.jar
-- classpath/embulk-util-aws-credentials-0.2.13.jar
+- classpath/embulk-input-s3-0.2.14.jar
+- classpath/embulk-util-aws-credentials-0.2.14.jar
 - classpath/httpclient-4.5.2.jar
 - classpath/httpcore-4.4.4.jar
 - classpath/ion-java-1.0.2.jar