embulk-input-s3 0.3.0 → 0.3.5

data/src/main/java/org/embulk/input/s3/explorer/S3SingleFileExplorer.java

@@ -0,0 +1,35 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+
+public class S3SingleFileExplorer extends S3FileExplorer
+{
+    private final String path;
+
+    public S3SingleFileExplorer(final String bucket, final AmazonS3 client, final RetryExecutor retryExecutor, final String path)
+    {
+        super(bucket, client, retryExecutor);
+        this.path = path;
+    }
+
+    @Override
+    public void addToBuilder(final FileList.Builder builder)
+    {
+        final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucketName, path);
+
+        final ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
+            @Override
+            public ObjectMetadata call()
+            {
+                return s3Client.getObjectMetadata(objectMetadataRequest);
+            }
+        }.executeWith(retryExecutor);
+
+        builder.add(path, objectMetadata.getContentLength());
+    }
+}

data/src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java

@@ -0,0 +1,70 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.apache.commons.lang3.StringUtils;
+import org.embulk.input.s3.DefaultRetryable;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.RetryExecutor;
+import org.slf4j.Logger;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
+public class S3TimeOrderPrefixFileExplorer extends S3PrefixFileExplorer
+{
+    private static final Logger LOGGER = Exec.getLogger(S3TimeOrderPrefixFileExplorer.class);
+
+    private final Optional<Date> from;
+    private final Date to;
+
+    private String lastPath;
+
+    private int numOfReq = 0;
+
+    public S3TimeOrderPrefixFileExplorer(final String bucket, final AmazonS3 client, final RetryExecutor retryExecutor,
+            final String pathPrefix, final boolean skipGlacierObjects, final Optional<Date> from, final Date to)
+    {
+        super(bucket, client, retryExecutor, pathPrefix, skipGlacierObjects);
+        this.from = from;
+        this.to = to;
+    }
+
+    @Override
+    public List<S3ObjectSummary> fetch()
+    {
+        ++numOfReq;
+
+        final ListObjectsRequest req = new ListObjectsRequest(bucketName, pathPrefix, lastPath, null, 1024);
+        final ObjectListing objectListing = new DefaultRetryable<ObjectListing>("Listing objects")
+        {
+            @Override
+            public ObjectListing call()
+            {
+                return s3Client.listObjects(req);
+            }
+        }.executeWith(retryExecutor);
+        lastPath = objectListing.getNextMarker();
+
+        return objectListing.getObjectSummaries()
+                .stream()
+                .filter(s3ObjectSummary -> s3ObjectSummary.getLastModified().before(to)
+                        && (!from.isPresent() || s3ObjectSummary.getLastModified().equals(from.get()) || s3ObjectSummary.getLastModified().after(from.get())))
+                .collect(Collectors.toList());
+    }
+
+    @Override
+    public boolean hasNext()
+    {
+        if (lastPath == null) {
+            LOGGER.info("The total number of LIST requests is {}{}.", numOfReq,
+                    numOfReq < 10 ? StringUtils.EMPTY : ". Clean up your s3 bucket to reduce the number of requests and improve the ingesting performance");
+            return false;
+        }
+        return true;
+    }
+}

data/src/main/java/org/embulk/input/s3/utils/DateUtils.java

@@ -0,0 +1,28 @@
+package org.embulk.input.s3.utils;
+
+import com.google.common.base.Joiner;
+import org.embulk.config.ConfigException;
+import org.joda.time.format.DateTimeFormat;
+
+import java.util.Date;
+import java.util.List;
+
+public class DateUtils
+{
+    public static Date parse(final String value, final List<String> supportedFormats)
+            throws ConfigException
+    {
+        for (final String fmt : supportedFormats) {
+            try {
+                return DateTimeFormat.forPattern(fmt).parseDateTime(value).toDate();
+            } catch (final IllegalArgumentException e) {
+                // ignorable exception
+            }
+        }
+        throw new ConfigException("Unsupported DateTime value: '" + value + "', supported formats: [" + Joiner.on(",").join(supportedFormats) + "]");
+    }
+
+    private DateUtils()
+    {
+    }
+}

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -1,16 +1,11 @@
 package org.embulk.input.s3;
 
 import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.Region;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
@@ -25,21 +20,15 @@ import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
-import org.mockito.Mockito;
 
-import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Optional;
 
 import static org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assume.assumeNotNull;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
 
 public class TestS3FileInputPlugin
 {
@@ -97,7 +86,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void useLastPath()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy().set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv");
         ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
@@ -117,7 +105,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void emptyFilesWithLastPath()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy()
                 .set("path_prefix", "empty_files_prefix")
@@ -130,7 +117,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void useTotalFileCountLimit()
-            throws Exception
     {
         ConfigSource config = this.config.deepCopy().set("total_file_count_limit", 0);
         ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
@@ -141,7 +127,6 @@ public class TestS3FileInputPlugin
 
     @Test
     public void usePathMatchPattern()
-            throws Exception
     {
         { // match pattern
             ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/sample_01");
@@ -227,44 +212,6 @@ public class TestS3FileInputPlugin
         assertEquals(s3Client.getRegion(), Region.US_Standard);
     }
 
-    @Test(expected = ConfigException.class)
-    public void useSkipGlacierObjects() throws Exception
-    {
-        AmazonS3 client;
-        client = mock(AmazonS3.class);
-        doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class));
-
-        AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS);
-        plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.empty(), false);
-    }
-
-    private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize)
-    {
-        FileList.Builder builder = new FileList.Builder(config);
-        for (int i = 0; i < nameAndSize.length; i += 2) {
-            builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
-        }
-        return builder;
-    }
-
-    private ObjectListing s3objectList(String key, StorageClass storageClass) throws Exception
-    {
-        ObjectListing list = new ObjectListing();
-
-        S3ObjectSummary element = new S3ObjectSummary();
-        element.setKey(key);
-        element.setStorageClass(storageClass.toString());
-
-        List<S3ObjectSummary> objectSummaries = new ArrayList<>();
-        objectSummaries.add(element);
-
-        Field field = list.getClass().getDeclaredField("objectSummaries");
-        field.setAccessible(true);
-        field.set(list, objectSummaries);
-
-        return list;
-    }
-
     static class Control
             implements InputPlugin.Control
     {

data/src/test/java/org/embulk/input/s3/explorer/TestS3NameOrderPrefixFileExplorer.java

@@ -0,0 +1,67 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Mock;
+import org.mockito.internal.util.reflection.FieldSetter;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3NameOrderPrefixFileExplorer
+{
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String PATH_PREFIX = "path_prefix";
+    private static final String LAST_PATH = "last_path";
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    private S3NameOrderPrefixFileExplorer s3NameOrderPrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3NameOrderPrefixFileExplorer = new S3NameOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, false, LAST_PATH);
+    }
+
+    @Test
+    public void fetch_should_return_list_objects()
+    {
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+
+        s3NameOrderPrefixFileExplorer.fetch();
+        final ArgumentCaptor<ListObjectsRequest> listObjectsRequestCaptor = ArgumentCaptor.forClass(ListObjectsRequest.class);
+
+        verify(ol).getNextMarker();
+        verify(s3Client).listObjects(listObjectsRequestCaptor.capture());
+        final ListObjectsRequest listObjectsRequest = listObjectsRequestCaptor.getValue();
+        assertEquals(BUCKET_NAME, listObjectsRequest.getBucketName());
+        assertEquals(PATH_PREFIX, listObjectsRequest.getPrefix());
+        assertEquals(LAST_PATH, listObjectsRequest.getMarker());
+    }
+
+    @Test
+    public void hasNext_should_return_false_if_no_lastpath() throws NoSuchFieldException
+    {
+        new FieldSetter(s3NameOrderPrefixFileExplorer, s3NameOrderPrefixFileExplorer.getClass().getDeclaredField("lastPath")).set(null);
+        assertFalse(s3NameOrderPrefixFileExplorer.hasNext());
+    }
+}

data/src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java

@@ -0,0 +1,128 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.StorageClass;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.input.s3.FileList;
+import org.embulk.spi.util.RetryExecutor;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3PrefixFileExplorer
+{
+    private static final String PATH_PREFIX = "path_prefix";
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String OBJECT_KEY = "key";
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    @Mock
+    private FileList.Builder builder;
+
+    @Mock
+    private S3ObjectSummary s3ObjectSummary;
+
+    private S3PrefixFileExplorer s3PrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, false);
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+    }
+
+    @Test(expected = ConfigException.class)
+    public void addToBuilder_should_throw_exception_if_notskipped_glacier_storage()
+    {
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
+        s3PrefixFileExplorer.addToBuilder(builder);
+    }
+
+    @Test
+    public void addToBuilder_should_skip_glacier_storage_if_allowed()
+    {
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Glacier.toString());
+        // override spied object for changing `skipGlacierObjects`
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, true);
+        doReturn(false).when(s3PrefixFileExplorer).hasNext();
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+        s3PrefixFileExplorer.addToBuilder(builder);
+
+        verify(s3PrefixFileExplorer).hasNext();
+        verify(s3ObjectSummary, never()).getSize();
+    }
+
+    @Test
+    public void addToBuilder_should_loop_till_nothing_left()
+    {
+        // There are 3 loops totally but only 2 keys have been imported because the first key is in Glacier storage class and is skipped
+        when(builder.needsMore()).thenReturn(true);
+        // override spied object for changing `skipGlacierObjects`
+        s3PrefixFileExplorer = spyS3PrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX, true);
+        when(s3ObjectSummary.getStorageClass())
+                .thenReturn(StorageClass.Glacier.toString())
+                .thenReturn(StorageClass.Standard.toString());
+        when(s3ObjectSummary.getSize()).thenReturn(1L);
+        when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
+        doReturn(Collections.singletonList(s3ObjectSummary)).when(s3PrefixFileExplorer).fetch();
+        doReturn(true).doReturn(true).doReturn(false).when(s3PrefixFileExplorer).hasNext();
+
+        s3PrefixFileExplorer.addToBuilder(builder);
+        verify(builder, times(2)).add(PATH_PREFIX + OBJECT_KEY, 1);
+    }
+
+    @Test
+    public void addToBuilder_should_stop_import_if_too_many_files()
+    {
+        when(builder.needsMore()).thenReturn(false);
+        when(s3ObjectSummary.getStorageClass()).thenReturn(StorageClass.Standard.toString());
+        when(s3ObjectSummary.getKey()).thenReturn(PATH_PREFIX + OBJECT_KEY);
+        when(s3ObjectSummary.getSize()).thenReturn(1L);
+        doReturn(true).when(s3PrefixFileExplorer).hasNext();
+        s3PrefixFileExplorer.addToBuilder(builder);
+
+        verify(builder).add(PATH_PREFIX + OBJECT_KEY, 1);
+        verify(s3PrefixFileExplorer, never()).hasNext();
+    }
+
+    private S3PrefixFileExplorer spyS3PrefixFileExplorer(final String bucketName, final AmazonS3 s3Client, final RetryExecutor retryExecutor, final String pathPrefix, final boolean skipGlacierObjects)
+    {
+        return spy(new S3PrefixFileExplorer(bucketName, s3Client, retryExecutor, pathPrefix, skipGlacierObjects)
+        {
+            @Override
+            protected List<S3ObjectSummary> fetch()
+            {
+                return null;
+            }
+
+            @Override
+            protected boolean hasNext()
+            {
+                return false;
+            }
+        });
+    }
+}