embulk-input-s3 0.2.21 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4bd7ca7dab8a4d3a6f25af6a33cf367965693e30
-  data.tar.gz: 4d3a8175e1d6ce32eaa780a44d8de66849ae34ca
+  metadata.gz: 7aef07b030009a6352551d4b7ebd78be174dbf53
+  data.tar.gz: ec9ecba65e22bc04e73819d145ccb2a6a4d8115e
 SHA512:
-  metadata.gz: e6aecfe225ac14c708a5f6acd7a605be5bca343f33af5379e1fa0b4a33219cf342f65976667d943b90da6c812f55a6f96680659db39752a32d91c1e3abeec496
-  data.tar.gz: 300e89205477ae9d01122fdda478b777b11605eb0302a9f2afee8ee536ef2a31b76f1001572800eca4f905452f8b894aad0bf6f3555bf5783e4bd816c3ce3151
+  metadata.gz: 867c41e167c9addeeabc893781c912880756b9ef5a16554a16440972acd30c425500b927a17736d2892cff167d64c352e11c87386808f756453290beb0b738a7
+  data.tar.gz: 07f0fea1f0716c3b74d384eb8174f6193d84e0714dabfd8fa4bf8cdaaba6b25f27f1a26aa5a1746115a89a095ae3799d3000350ffdf890be4255bed29a45d369

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -17,7 +17,6 @@ import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
 import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -32,6 +31,7 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
 import org.embulk.spi.util.ResumableInputStream;
 import org.embulk.spi.util.RetryExecutor;
 import org.embulk.util.aws.credentials.AwsCredentials;
@@ -42,7 +42,9 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Optional;
 
+import static java.lang.String.format;
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public abstract class AbstractS3FileInputPlugin
@@ -54,46 +56,46 @@ public abstract class AbstractS3FileInputPlugin
             extends AwsCredentialsTask, FileList.Task, RetrySupportPluginTask, Task
     {
         @Config("bucket")
-        public String getBucket();
+        String getBucket();
 
         @Config("path_prefix")
         @ConfigDefault("null")
-        public Optional<String> getPathPrefix();
+        Optional<String> getPathPrefix();
 
         @Config("path")
         @ConfigDefault("null")
-        public Optional<String> getPath();
+        Optional<String> getPath();
 
         @Config("last_path")
         @ConfigDefault("null")
-        public Optional<String> getLastPath();
+        Optional<String> getLastPath();
 
         @Config("access_key_id")
         @ConfigDefault("null")
-        public Optional<String> getAccessKeyId();
+        Optional<String> getAccessKeyId();
 
         @Config("http_proxy")
         @ConfigDefault("null")
-        public Optional<HttpProxy> getHttpProxy();
+        Optional<HttpProxy> getHttpProxy();
 
-        public void setHttpProxy(Optional<HttpProxy> httpProxy);
+        void setHttpProxy(Optional<HttpProxy> httpProxy);
 
         @Config("incremental")
         @ConfigDefault("true")
-        public boolean getIncremental();
+        boolean getIncremental();
 
         @Config("skip_glacier_objects")
         @ConfigDefault("false")
-        public boolean getSkipGlacierObjects();
+        boolean getSkipGlacierObjects();
 
         // TODO timeout, ssl, etc
 
-        public FileList getFiles();
+        FileList getFiles();
 
-        public void setFiles(FileList files);
+        void setFiles(FileList files);
 
         @ConfigInject
-        public BufferAllocator getBufferAllocator();
+        BufferAllocator getBufferAllocator();
     }
 
     protected abstract Class<? extends PluginTask> getTaskClass();
@@ -129,7 +131,7 @@ public abstract class AbstractS3FileInputPlugin
         // last_path
         if (task.getIncremental()) {
             Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
-            LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orNull());
+            LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orElse(""));
             configDiff.set("last_path", lastPath);
         }
         return configDiff;
@@ -335,7 +337,7 @@ public abstract class AbstractS3FileInputPlugin
                                            boolean skipGlacierObjects,
                                            RetryExecutor retryExec)
     {
-        String lastKey = lastPath.orNull();
+        String lastKey = lastPath.orElse(null);
         do {
             final String finalLastKey = lastKey;
             final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
@@ -402,10 +404,10 @@ public abstract class AbstractS3FileInputPlugin
         @Override
         public InputStream reopen(final long offset, final Exception closedCause) throws IOException
         {
-            log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+            log.warn(format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
             request.setRange(offset, contentLength - 1);  // [first, last]
 
-            return new DefaultRetryable<S3ObjectInputStream>("Opening the file") {
+            return new DefaultRetryable<S3ObjectInputStream>(format("Getting object '%s'", request.getKey())) {
                 @Override
                 public S3ObjectInputStream call()
                 {
@@ -457,17 +459,25 @@ public abstract class AbstractS3FileInputPlugin
         }
 
         @Override
-        public InputStream openNext() throws IOException
+        public InputStreamWithHints openNextWithHints() throws IOException
         {
             if (!iterator.hasNext()) {
                 return null;
             }
-            String key = iterator.next();
-            GetObjectRequest request = new GetObjectRequest(bucket, key);
-            S3Object obj = client.getObject(request);
-            long objectSize = obj.getObjectMetadata().getContentLength();
-            LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
-            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec));
+            final String key = iterator.next();
+            final GetObjectRequest request = new GetObjectRequest(bucket, key);
+
+            S3Object object = new DefaultRetryable<S3Object>(format("Getting object '%s'", request.getKey())) {
+                @Override
+                public S3Object call()
+                {
+                    return client.getObject(request);
+                }
+            }.executeWithCheckedException(retryExec, IOException.class);
+
+            long objectSize = object.getObjectMetadata().getContentLength();
+            InputStream inputStream = new ResumableInputStream(object.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec));
+            return new InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key));
         }
 
         @Override

data/src/main/java/org/embulk/input/s3/FileList.java

@@ -3,7 +3,6 @@ package org.embulk.input.s3;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 
 import org.embulk.config.Config;
@@ -23,6 +22,7 @@ import java.nio.charset.StandardCharsets;
 import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 import java.util.regex.Pattern;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
@@ -179,7 +179,7 @@ public class FileList
             catch (IOException ex) {
                 throw Throwables.propagate(ex);
             }
-            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.ofNullable(last));
         }
 
         private List<List<Entry>> getSplits(List<Entry> all)

data/src/main/java/org/embulk/input/s3/HttpProxy.java

@@ -1,10 +1,11 @@
 package org.embulk.input.s3;
 
-import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.Task;
 
+import java.util.Optional;
+
 /**
  * HttpProxy is config unit for Input/Output plugins' configs.
  *
@@ -15,21 +16,21 @@ public interface HttpProxy
     extends Task
 {
     @Config("host")
-    public String getHost();
+    String getHost();
 
     @Config("port")
     @ConfigDefault("null")
-    public Optional<Integer> getPort();
+    Optional<Integer> getPort();
 
     @Config("https")
     @ConfigDefault("true")
-    public boolean getHttps();
+    boolean getHttps();
 
     @Config("user")
     @ConfigDefault("null")
-    public Optional<String> getUser();
+    Optional<String> getUser();
 
     @Config("password")
     @ConfigDefault("null")
-    public Optional<String> getPassword();
+    Optional<String> getPassword();
 }

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -3,12 +3,13 @@ package org.embulk.input.s3;
 import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration;
 import com.amazonaws.services.s3.AmazonS3;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
-import com.google.common.base.Optional;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
+import java.util.Optional;
+
 public class S3FileInputPlugin
         extends AbstractS3FileInputPlugin
 {
@@ -17,11 +18,11 @@ public class S3FileInputPlugin
     {
         @Config("endpoint")
         @ConfigDefault("null")
-        public Optional<String> getEndpoint();
+        Optional<String> getEndpoint();
 
         @Config("region")
         @ConfigDefault("null")
-        public Optional<String> getRegion();
+        Optional<String> getRegion();
     }
 
     private static final Logger log = Exec.getLogger(S3FileInputPlugin.class);

data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java

@@ -6,7 +6,6 @@ import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.google.common.base.Optional;
 import org.apache.http.HttpStatus;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.spi.util.RetryExecutor;
@@ -14,6 +13,8 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;

data/src/test/java/org/embulk/input/s3/TestHttpProxy.java

@@ -1,6 +1,5 @@
 package org.embulk.input.s3;
 
-import com.google.common.base.Optional;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigSource;
 import org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
@@ -8,6 +7,8 @@ import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 
+import java.util.Optional;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
 
@@ -41,7 +42,7 @@ public class TestHttpProxy
             String host = "my_host";
             ConfigSource conf = config.deepCopy().set("host", host);
             HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
-            assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
+            assertHttpProxy(host, Optional.empty(), true, Optional.empty(), Optional.empty(),
                     httpProxy);
         }
 
@@ -51,7 +52,7 @@ public class TestHttpProxy
                     .set("host", host)
                     .set("https", true);
             HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
-            assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
+            assertHttpProxy(host, Optional.empty(), true, Optional.empty(), Optional.empty(),
                     httpProxy);
         }
 
@@ -61,7 +62,7 @@ public class TestHttpProxy
                     .set("host", host)
                     .set("https", false);
             HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
-            assertHttpProxy(host, Optional.<Integer>absent(), false, Optional.<String>absent(), Optional.<String>absent(),
+            assertHttpProxy(host, Optional.empty(), false, Optional.empty(), Optional.empty(),
                     httpProxy);
         }
 
@@ -72,7 +73,7 @@ public class TestHttpProxy
                     .set("host", host)
                     .set("port", port);
             HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
-            assertHttpProxy(host, Optional.of(port), true, Optional.<String>absent(), Optional.<String>absent(),
+            assertHttpProxy(host, Optional.of(port), true, Optional.empty(), Optional.empty(),
                     httpProxy);
         }
 

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -6,7 +6,6 @@ import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.Region;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
 import com.amazonaws.services.s3.model.StorageClass;
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
@@ -31,6 +30,7 @@ import org.mockito.Mockito;
 import java.lang.reflect.Field;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Optional;
 
 import static org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
 import static org.junit.Assert.assertEquals;
@@ -235,7 +235,7 @@ public class TestS3FileInputPlugin
         doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class));
 
         AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS);
-        plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.<String>absent(), false);
+        plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.empty(), false);
     }
 
     private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.21
+  version: 0.3.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-08 00:00:00.000000000 Z
+date: 2018-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -61,21 +61,19 @@ files:
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
 - src/test/resources/sample_01.csv
-- classpath/aws-java-sdk-kms-1.11.253.jar
-- classpath/embulk-input-s3-0.2.21.jar
-- classpath/jackson-core-2.6.7.jar
+- classpath/embulk-util-aws-credentials-0.3.0.jar
+- classpath/httpcore-4.4.9.jar
+- classpath/httpclient-4.5.5.jar
 - classpath/ion-java-1.0.2.jar
-- classpath/aws-java-sdk-s3-1.11.253.jar
-- classpath/httpclient-4.5.2.jar
-- classpath/commons-codec-1.9.jar
-- classpath/jmespath-java-1.11.253.jar
+- classpath/embulk-input-s3-0.3.0.jar
+- classpath/aws-java-sdk-core-1.11.466.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
-- classpath/aws-java-sdk-core-1.11.253.jar
-- classpath/embulk-util-aws-credentials-0.2.21.jar
+- classpath/commons-codec-1.10.jar
+- classpath/jmespath-java-1.11.466.jar
+- classpath/jackson-databind-2.6.7.2.jar
 - classpath/jackson-dataformat-cbor-2.6.7.jar
-- classpath/jackson-databind-2.6.7.1.jar
-- classpath/httpcore-4.4.4.jar
-- classpath/jackson-annotations-2.6.0.jar
+- classpath/aws-java-sdk-s3-1.11.466.jar
+- classpath/aws-java-sdk-kms-1.11.466.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:
 - Apache 2.0