embulk-input-s3 0.3.0 → 0.3.5

data/src/test/java/org/embulk/input/s3/explorer/TestS3SingleFileExplorer.java

@@ -0,0 +1,56 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.input.s3.FileList;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3SingleFileExplorer
+{
+    private static final String PATH = "path";
+    private static final String BUCKET_NAME = "bucket_name";
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime embulkTestRuntime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    @Mock
+    private FileList.Builder builder;
+
+    @Mock
+    private ObjectMetadata metadata;
+
+    private S3SingleFileExplorer s3SingleFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        s3SingleFileExplorer = new S3SingleFileExplorer(BUCKET_NAME, s3Client, null, PATH);
+    }
+
+    @Test
+    public void addToBuilder_should_request_single_object_metadata()
+    {
+        when(s3Client.getObjectMetadata(any(GetObjectMetadataRequest.class))).thenReturn(metadata);
+        when(metadata.getContentLength()).thenReturn(1L);
+        s3SingleFileExplorer.addToBuilder(builder);
+
+        verify(builder).add(PATH, 1);
+    }
+}

data/src/test/java/org/embulk/input/s3/explorer/TestS3TimeOrderPrefixFileExplorer.java

@@ -0,0 +1,112 @@
+package org.embulk.input.s3.explorer;
+
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import org.embulk.EmbulkTestRuntime;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.Mock;
+import org.mockito.internal.util.reflection.FieldSetter;
+import org.mockito.runners.MockitoJUnitRunner;
+
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.class)
+public class TestS3TimeOrderPrefixFileExplorer
+{
+    private static final String BUCKET_NAME = "bucket_name";
+    private static final String PATH_PREFIX = "path_prefix";
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Mock
+    private AmazonS3 s3Client;
+
+    private S3TimeOrderPrefixFileExplorer s3TimeOrderPrefixFileExplorer;
+
+    @Before
+    public void setUp()
+    {
+        final Calendar cal = Calendar.getInstance();
+        cal.set(2019, Calendar.MAY, 25, 10, 0);
+        s3TimeOrderPrefixFileExplorer = new S3TimeOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX,
+                false, Optional.empty(), cal.getTime());
+    }
+
+    @Test
+    public void fetch_should_return_filtered_objects_before_end_time()
+    {
+        final S3ObjectSummary s3ObjectBefore = mock(S3ObjectSummary.class);
+        final Calendar cal = Calendar.getInstance();
+        cal.set(2019, Calendar.MAY, 24, 10, 0);
+        when(s3ObjectBefore.getLastModified()).thenReturn(cal.getTime());
+
+        final S3ObjectSummary s3ObjectAfter = mock(S3ObjectSummary.class);
+        cal.set(2019, Calendar.MAY, 26, 10, 0);
+        when(s3ObjectAfter.getLastModified()).thenReturn(cal.getTime());
+
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+        when(ol.getObjectSummaries()).thenReturn(Arrays.asList(s3ObjectBefore, s3ObjectAfter));
+
+        final List<S3ObjectSummary> result = s3TimeOrderPrefixFileExplorer.fetch();
+        assertEquals(1, result.size());
+        assertEquals(s3ObjectBefore, result.get(0));
+    }
+
+    @Test
+    public void fetch_should_return_filtered_objects_after_or_equals_begin_time()
+    {
+        final Calendar to = Calendar.getInstance();
+        to.set(2019, Calendar.MAY, 25, 10, 0);
+        final Calendar from = Calendar.getInstance();
+        from.set(2019, Calendar.MAY, 24, 10, 0);
+        s3TimeOrderPrefixFileExplorer = new S3TimeOrderPrefixFileExplorer(BUCKET_NAME, s3Client, null, PATH_PREFIX,
+                false, Optional.of(from.getTime()), to.getTime());
+
+        final S3ObjectSummary s3ObjectEqual = mock(S3ObjectSummary.class);
+        final Calendar equalCal = Calendar.getInstance();
+        equalCal.set(2019, Calendar.MAY, 24, 10, 0);
+        when(s3ObjectEqual.getLastModified()).thenReturn(equalCal.getTime());
+
+        final S3ObjectSummary s3ObjectBefore = mock(S3ObjectSummary.class);
+        final Calendar beforeCal = Calendar.getInstance();
+        beforeCal.set(2019, Calendar.MAY, 24, 20, 0);
+        when(s3ObjectBefore.getLastModified()).thenReturn(beforeCal.getTime());
+
+        final S3ObjectSummary s3ObjectAfter = mock(S3ObjectSummary.class);
+        final Calendar afterCal = Calendar.getInstance();
+        afterCal.set(2019, Calendar.MAY, 26, 10, 0);
+        when(s3ObjectAfter.getLastModified()).thenReturn(afterCal.getTime());
+
+        final ObjectListing ol = mock(ObjectListing.class);
+        when(s3Client.listObjects(any(ListObjectsRequest.class))).thenReturn(ol);
+        when(ol.getObjectSummaries()).thenReturn(Arrays.asList(s3ObjectEqual, s3ObjectBefore, s3ObjectAfter));
+
+        final List<S3ObjectSummary> result = s3TimeOrderPrefixFileExplorer.fetch();
+        assertEquals(2, result.size());
+        assertEquals(s3ObjectEqual, result.get(0));
+        assertEquals(s3ObjectBefore, result.get(1));
+    }
+
+    @Test
+    public void hasNext_should_return_false_if_no_lastpath() throws NoSuchFieldException
+    {
+        new FieldSetter(s3TimeOrderPrefixFileExplorer, s3TimeOrderPrefixFileExplorer.getClass().getDeclaredField("lastPath")).set(null);
+        assertFalse(s3TimeOrderPrefixFileExplorer.hasNext());
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-26 00:00:00.000000000 Z
+date: 2019-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -53,19 +53,28 @@ files:
 - src/main/java/org/embulk/input/s3/HttpProxy.java
 - src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
-- src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java
+- src/main/java/org/embulk/input/s3/explorer/S3FileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3NameOrderPrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3PrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3SingleFileExplorer.java
+- src/main/java/org/embulk/input/s3/explorer/S3TimeOrderPrefixFileExplorer.java
+- src/main/java/org/embulk/input/s3/utils/DateUtils.java
 - src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestDefaultRetryable.java
 - src/test/java/org/embulk/input/s3/TestFileList.java
 - src/test/java/org/embulk/input/s3/TestHttpProxy.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3NameOrderPrefixFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3PrefixFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3SingleFileExplorer.java
+- src/test/java/org/embulk/input/s3/explorer/TestS3TimeOrderPrefixFileExplorer.java
 - src/test/resources/sample_01.csv
-- classpath/embulk-util-aws-credentials-0.3.0.jar
+- classpath/embulk-util-aws-credentials-0.3.5.jar
 - classpath/httpcore-4.4.9.jar
 - classpath/httpclient-4.5.5.jar
+- classpath/embulk-input-s3-0.3.5.jar
 - classpath/ion-java-1.0.2.jar
-- classpath/embulk-input-s3-0.3.0.jar
 - classpath/aws-java-sdk-core-1.11.466.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
 - classpath/commons-codec-1.10.jar
@@ -73,6 +82,7 @@ files:
 - classpath/jackson-databind-2.6.7.2.jar
 - classpath/jackson-dataformat-cbor-2.6.7.jar
 - classpath/aws-java-sdk-s3-1.11.466.jar
+- classpath/aws-java-sdk-sts-1.11.466.jar
 - classpath/aws-java-sdk-kms-1.11.466.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:

data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java

@@ -1,164 +0,0 @@
-package org.embulk.input.s3;
-
-import com.amazonaws.AmazonServiceException;
-import com.amazonaws.services.s3.AmazonS3;
-import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import org.apache.http.HttpStatus;
-import org.embulk.EmbulkTestRuntime;
-import org.embulk.spi.util.RetryExecutor;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-
-import java.util.Optional;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-
-public class TestAbstractS3FileInputPlugin
-{
-    private static RetryExecutor retryExecutor()
-    {
-        return RetryExecutor.retryExecutor()
-                .withInitialRetryWait(0)
-                .withMaxRetryWait(0);
-    }
-
-    private static AbstractS3FileInputPlugin dummyS3Plugin()
-    {
-        return new AbstractS3FileInputPlugin()
-        {
-            @Override
-            protected Class<? extends PluginTask> getTaskClass()
-            {
-                return PluginTask.class;
-            }
-        };
-    }
-
-    private static class SomeException extends RuntimeException
-    {
-    }
-
-    @Rule
-    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
-
-    private AmazonS3 client;
-
-    @Before
-    public void createResources()
-    {
-        client = mock(AmazonS3.class);
-    }
-
-    @Test
-    public void listS3FilesByPrefix()
-    {
-        doReturn(new ObjectListing()).when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true);
-    }
-
-    @Test
-    public void listS3FileByPrefix_with_retry()
-    {
-        doThrow(new RuntimeException()).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = SomeException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception()
-    {
-        doThrow(new SomeException()).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(0));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("Forbidden exception");
-        exception.setStatusCode(HttpStatus.SC_FORBIDDEN);
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("method not allow exception");
-        exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED);
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test(expected = AmazonServiceException.class)
-    public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code()
-    {
-        AmazonServiceException exception = new AmazonServiceException("expired token exception");
-        exception.setStatusCode(HttpStatus.SC_BAD_REQUEST);
-        exception.setErrorCode("ExpiredToken");
-        exception.setErrorType(AmazonServiceException.ErrorType.Client);
-
-        doThrow(exception).doReturn(new ObjectListing())
-                .when(client).listObjects(any(ListObjectsRequest.class));
-        FileList.Builder builder = new FileList.Builder();
-        dummyS3Plugin().listS3FilesByPrefix(
-                builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
-                retryExecutor().withRetryLimit(1));
-    }
-
-    @Test
-    public void addS3DirectObject()
-    {
-        doReturn(new ObjectMetadata()).when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(builder, client, "some_bucket", "some_prefix");
-    }
-
-    @Test
-    public void addS3DirectObject_with_retry()
-    {
-        doThrow(new RuntimeException()).doReturn(new ObjectMetadata())
-                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(
-                builder, client, "some_bucket", "some_prefix",
-                retryExecutor());
-    }
-
-    @Test(expected = SomeException.class)
-    public void addS3DirectObject_on_retry_gave_up_should_throw_original_exception()
-    {
-        doThrow(new SomeException()).doReturn(new ObjectMetadata())
-                .when(client).getObjectMetadata(any(GetObjectMetadataRequest.class));
-        FileList.Builder builder = new FileList.Builder().pathMatchPattern("");
-        dummyS3Plugin().addS3DirectObject(
-                builder, client, "some_bucket", "some_prefix",
-                retryExecutor().withRetryLimit(0));
-    }
-}