embulk-input-s3 0.1.7 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cd67a9aea3d1352358739b2c6917f3204301443e
-  data.tar.gz: b143676db2502a2e39d7cb430bd4bf8f67933a5b
+  metadata.gz: 02cd2ac88b83d92846715d194928f38a32693ebe
+  data.tar.gz: 145815003f894fc2a2a224266e00908dae89cf79
 SHA512:
-  metadata.gz: 08b84503dda64c6d4e3765d66a368070d47bf1214d96ac9f5ff77478cc888782c799fb193a0925ce508b8b0867ead0e2bde4185db2f92eb0433807c400cd18d6
-  data.tar.gz: b6541d55d9e4b7317267b3f57eaed9795c0b1ff7591f4de240890e2b5f0580575b39461e066fbf85bef5e894e54c201641721195899b9246f50f79756e703239
+  metadata.gz: e9d81495964337639a2a6e7de2ce9ca3c3e5b39ca869ff1873e5ed119de95ac8ae778fe3450d85068dcd7b773bbfd8345835b46ab226ad73f1117c2fb614c0bc
+  data.tar.gz: 5f50eb790d085fb427a23e3819bdada69bb0cafba84e6e4175a8a317e7cf7e37e0a675866a87b893600c6ccfeb59654662e452c56d93a556d5559319c2760543

data/build.gradle

@@ -1,41 +1,3 @@
-plugins {
-    id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
-    id "java"
-}
-import com.github.jrubygradle.JRubyExec
-repositories {
-    mavenCentral()
-    mavenLocal()
-    jcenter()
-}
-configurations {
-    provided
-}
-
-version = "0.1.7"
-
-dependencies {
-    compile  "org.embulk:embulk-core:0.5.3"
-    provided "org.embulk:embulk-core:0.5.3"
-    compile "com.amazonaws:aws-java-sdk-s3:1.9.22"
-    testCompile "junit:junit:4.+"
-    testCompile "org.mockito:mockito-core:1.+"
-}
-
-task classpath(type: Copy, dependsOn: ["jar"]) {
-    doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided + files(jar.archivePath))
-    into "classpath"
-}
-clean { delete 'classpath' }
-
-task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "build/gemspec"
-    doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
-}
-
 task gemspec << { file("build/gemspec").write($/
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -0,0 +1,318 @@
+package org.embulk.input.s3;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.io.InputStream;
+import com.google.common.collect.ImmutableList;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import org.slf4j.Logger;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigInject;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.CommitReport;
+import org.embulk.spi.BufferAllocator;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.ResumableInputStream;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
+public abstract class AbstractS3FileInputPlugin
+        implements FileInputPlugin
+{
+    public interface PluginTask
+            extends Task
+    {
+        @Config("bucket")
+        public String getBucket();
+
+        @Config("path_prefix")
+        public String getPathPrefix();
+
+        @Config("last_path")
+        @ConfigDefault("null")
+        public Optional<String> getLastPath();
+
+        @Config("access_key_id")
+        public String getAccessKeyId();
+
+        @Config("secret_access_key")
+        public String getSecretAccessKey();
+
+        // TODO timeout, ssl, etc
+
+        // TODO support more options such as STS
+
+        public List<String> getFiles();
+        public void setFiles(List<String> files);
+
+        @ConfigInject
+        public BufferAllocator getBufferAllocator();
+    }
+
+    protected abstract Class<? extends PluginTask> getTaskClass();
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(getTaskClass());
+
+        // list files recursively
+        task.setFiles(listFiles(task));
+
+        // number of processors is same with number of files
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+            int taskCount,
+            FileInputPlugin.Control control)
+    {
+        PluginTask task = taskSource.loadTask(getTaskClass());
+
+        // validate task
+        newS3Client(task);
+
+        control.run(taskSource, taskCount);
+
+        // build next config
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        // last_path
+        if (task.getFiles().isEmpty()) {
+            // keep the last value
+            if (task.getLastPath().isPresent()) {
+                configDiff.set("last_path", task.getLastPath().get());
+            }
+        } else {
+            List<String> files = new ArrayList<String>(task.getFiles());
+            Collections.sort(files);
+            configDiff.set("last_path", files.get(files.size() - 1));
+        }
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+            int taskCount,
+            List<CommitReport> successCommitReports)
+    {
+        // do nothing
+    }
+
+    protected AmazonS3Client newS3Client(PluginTask task)
+    {
+        return new AmazonS3Client(getCredentialsProvider(task), getClientConfiguration(task));
+    }
+
+    protected AWSCredentialsProvider getCredentialsProvider(PluginTask task)
+    {
+        final AWSCredentials cred = new BasicAWSCredentials(
+                task.getAccessKeyId(), task.getSecretAccessKey());
+        return new AWSCredentialsProvider() {
+            public AWSCredentials getCredentials()
+            {
+                return cred;
+            }
+
+            public void refresh()
+            {
+            }
+        };
+    }
+
+    protected ClientConfiguration getClientConfiguration(PluginTask task)
+    {
+        ClientConfiguration clientConfig = new ClientConfiguration();
+
+        //clientConfig.setProtocol(Protocol.HTTP);
+        clientConfig.setMaxConnections(50); // SDK default: 50
+        clientConfig.setMaxErrorRetry(3); // SDK default: 3
+        clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
+
+        return clientConfig;
+    }
+
+    private List<String> listFiles(PluginTask task)
+    {
+        AmazonS3Client client = newS3Client(task);
+        String bucketName = task.getBucket();
+
+        return listS3FilesByPrefix(client, bucketName, task.getPathPrefix(), task.getLastPath());
+    }
+
+    /**
+     * Lists S3 filenames filtered by prefix.
+     *
+     * The resulting list does not include the file that's size == 0.
+     */
+    public static List<String> listS3FilesByPrefix(AmazonS3Client client, String bucketName,
+            String prefix, Optional<String> lastPath)
+    {
+        ImmutableList.Builder<String> builder = ImmutableList.builder();
+
+        String lastKey = lastPath.orNull();
+        do {
+            ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
+            ObjectListing ol = client.listObjects(req);
+            for(S3ObjectSummary s : ol.getObjectSummaries()) {
+                if (s.getSize() > 0) {
+                    builder.add(s.getKey());
+                }
+            }
+            lastKey = ol.getNextMarker();
+        } while(lastKey != null);
+
+        return builder.build();
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(getTaskClass());
+        return new S3FileInput(task, taskIndex);
+    }
+
+    private static class S3InputStreamReopener
+            implements ResumableInputStream.Reopener
+    {
+        private final Logger log = Exec.getLogger(S3InputStreamReopener.class);
+
+        private final AmazonS3Client client;
+        private final GetObjectRequest request;
+        private final long contentLength;
+
+        public S3InputStreamReopener(AmazonS3Client client, GetObjectRequest request, long contentLength)
+        {
+            this.client = client;
+            this.request = request;
+            this.contentLength = contentLength;
+        }
+
+        @Override
+        public InputStream reopen(final long offset, final Exception closedCause) throws IOException
+        {
+            try {
+                return retryExecutor()
+                    .withRetryLimit(3)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30*1000)
+                    .runInterruptible(new Retryable<InputStream>() {
+                        @Override
+                        public InputStream call() throws InterruptedIOException
+                        {
+                            log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+                            request.setRange(offset, contentLength - 1);  // [first, last]
+                            return client.getObject(request).getObjectContent();
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;  // TODO
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            } else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+            } catch (RetryGiveupException ex) {
+                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
+                throw Throwables.propagate(ex.getCause());
+            } catch (InterruptedException ex) {
+                throw new InterruptedIOException();
+            }
+        }
+    }
+
+    public class S3FileInput
+            extends InputStreamFileInput
+            implements TransactionalFileInput
+    {
+        public S3FileInput(PluginTask task, int taskIndex)
+        {
+            super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
+        }
+
+        public void abort() { }
+
+        public CommitReport commit()
+        {
+            return Exec.newCommitReport();
+        }
+
+        @Override
+        public void close() { }
+    }
+
+    // TODO create single-file InputStreamFileInput utility
+    private class SingleFileProvider
+            implements InputStreamFileInput.Provider
+    {
+        private AmazonS3Client client;
+        private final String bucket;
+        private final String key;
+        private boolean opened = false;
+
+        public SingleFileProvider(PluginTask task, int taskIndex)
+        {
+            this.client = newS3Client(task);
+            this.bucket = task.getBucket();
+            this.key = task.getFiles().get(taskIndex);
+        }
+
+        @Override
+        public InputStream openNext() throws IOException
+        {
+            if (opened) {
+                return null;
+            }
+            opened = true;
+            GetObjectRequest request = new GetObjectRequest(bucket, key);
+            S3Object obj = client.getObject(request);
+            return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, obj.getObjectMetadata().getContentLength()));
+        }
+
+        @Override
+        public void close() { }
+    }
+}

data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java

@@ -1,327 +1,39 @@
 package org.embulk.input.s3;
 
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.InputStream;
-import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.slf4j.Logger;
-import com.amazonaws.auth.AWSCredentials;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
 import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.CommitReport;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
-import org.embulk.input.s3.RetryExecutor.Retryable;
-import org.embulk.input.s3.RetryExecutor.RetryGiveupException;
-import static org.embulk.input.s3.RetryExecutor.retryExecutor;
+import org.embulk.input.s3.AbstractS3FileInputPlugin;
 
 public class S3FileInputPlugin
-        implements FileInputPlugin
+        extends AbstractS3FileInputPlugin
 {
-    public interface PluginTask
-            extends Task
+    public interface S3PluginTask
+            extends PluginTask
     {
-        @Config("bucket")
-        public String getBucket();
-
-        @Config("path_prefix")
-        public String getPathPrefix();
-
-        @Config("last_path")
-        @ConfigDefault("null")
-        public Optional<String> getLastPath();
-
         @Config("endpoint")
         @ConfigDefault("null")
         public Optional<String> getEndpoint();
-
-        // TODO timeout, ssl, etc
-
-        @Config("access_key_id")
-        public String getAccessKeyId();
-
-        @Config("secret_access_key")
-        public String getSecretAccessKey();
-
-        // TODO support more options such as STS
-
-        public List<String> getFiles();
-        public void setFiles(List<String> files);
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
     }
 
     @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    protected Class<? extends PluginTask> getTaskClass()
     {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // list files recursively
-        task.setFiles(listFiles(task));
-
-        // TODO what if task.getFiles().isEmpty()?
-
-        // number of processors is same with number of files
-        return resume(task.dump(), task.getFiles().size(), control);
+        return S3PluginTask.class;
     }
 
     @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control)
+    protected AmazonS3Client newS3Client(PluginTask task)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-
-        control.run(taskSource, taskCount);
+        S3PluginTask t = (S3PluginTask) task;
 
-        // build next config
-        ConfigDiff configDiff = Exec.newConfigDiff();
+        AmazonS3Client client = super.newS3Client(t);
 
-        // last_path
-        if (task.getFiles().isEmpty()) {
-            // keep the last value
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
-            }
-        } else {
-            List<String> files = new ArrayList<String>(task.getFiles());
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
+        if (t.getEndpoint().isPresent()) {
+            client.setEndpoint(t.getEndpoint().get());
         }
 
-        return configDiff;
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<CommitReport> successCommitReports)
-    {
-        // do nothing
-    }
-
-    public static AWSCredentialsProvider getCredentialsProvider(PluginTask task)
-    {
-        final AWSCredentials cred = new BasicAWSCredentials(
-                task.getAccessKeyId(), task.getSecretAccessKey());
-        return new AWSCredentialsProvider() {
-            public AWSCredentials getCredentials()
-            {
-                return cred;
-            }
-
-            public void refresh()
-            {
-            }
-        };
-    }
-
-    private static AmazonS3Client newS3Client(PluginTask task)
-    {
-        AWSCredentialsProvider credentials = getCredentialsProvider(task);
-        AmazonS3Client client = newS3Client(credentials, task.getEndpoint());
         return client;
     }
-
-    private static AmazonS3Client newS3Client(AWSCredentialsProvider credentials,
-            Optional<String> endpoint)
-    {
-        // TODO get config from AmazonS3Task
-        ClientConfiguration clientConfig = new ClientConfiguration();
-        //clientConfig.setProtocol(Protocol.HTTP);
-        clientConfig.setMaxConnections(50); // SDK default: 50
-        clientConfig.setMaxErrorRetry(3); // SDK default: 3
-        clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
-
-        AmazonS3Client client = new AmazonS3Client(credentials, clientConfig);
-
-        if (endpoint.isPresent()) {
-            client.setEndpoint(endpoint.get());
-        }
-
-        return client;
-    }
-
-    public List<String> listFiles(PluginTask task)
-    {
-        AmazonS3Client client = newS3Client(task);
-        String bucketName = task.getBucket();
-
-        return listS3FilesByPrefix(client, bucketName, task.getPathPrefix(), task.getLastPath());
-    }
-
-    /**
-     * Lists S3 filenames filtered by prefix.
-     *
-     * The resulting list does not include the file that's size == 0.
-     */
-    public static List<String> listS3FilesByPrefix(AmazonS3Client client, String bucketName,
-            String prefix, Optional<String> lastPath)
-    {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-
-        String lastKey = lastPath.orNull();
-        do {
-            ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
-            ObjectListing ol = client.listObjects(req);
-            for(S3ObjectSummary s : ol.getObjectSummaries()) {
-                if (s.getSize() > 0) {
-                    builder.add(s.getKey());
-                }
-            }
-            lastKey = ol.getNextMarker();
-        } while(lastKey != null);
-
-        return builder.build();
-    }
-
-    @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        return new S3FileInput(task, taskIndex);
-    }
-
-    private static class S3RetryableOpener
-            implements RetryableInputStream.Opener
-    {
-        private final Logger log = Exec.getLogger(S3FileInputPlugin.class);
-
-        private final AmazonS3Client client;
-        private final GetObjectRequest request;
-        private final long contentLength;
-
-        public S3RetryableOpener(AmazonS3Client client, GetObjectRequest request, long contentLength)
-        {
-            this.client = client;
-            this.request = request;
-            this.contentLength = contentLength;
-        }
-
-        @Override
-        public InputStream open(final long offset, final Exception exception) throws IOException
-        {
-            try {
-                return retryExecutor()
-                    .withRetryLimit(3)
-                    .withInitialRetryWait(500)
-                    .withMaxRetryWait(30*1000)
-                    .runInterruptible(new Retryable<InputStream>() {
-                        @Override
-                        public InputStream call() throws InterruptedIOException
-                        {
-                            log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), exception);
-                            request.setRange(offset, contentLength - 1);  // [first, last]
-                            return client.getObject(request).getObjectContent();
-                        }
-
-                        @Override
-                        public boolean isRetryableException(Exception exception)
-                        {
-                            return true;  // TODO
-                        }
-
-                        @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                                throws RetryGiveupException
-                        {
-                            String message = String.format("S3 GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
-                            if (retryCount % 3 == 0) {
-                                log.warn(message, exception);
-                            } else {
-                                log.warn(message);
-                            }
-                        }
-
-                        @Override
-                        public void onGiveup(Exception firstException, Exception lastException)
-                                throws RetryGiveupException
-                        {
-                        }
-                    });
-            } catch (RetryGiveupException ex) {
-                Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
-                throw Throwables.propagate(ex.getCause());
-            } catch (InterruptedException ex) {
-                throw new InterruptedIOException();
-            }
-        }
-    }
-
-    public static class S3FileInput
-            extends InputStreamFileInput
-            implements TransactionalFileInput
-    {
-        // TODO create single-file InputStreamFileInput utility
-        private static class SingleFileProvider
-                implements InputStreamFileInput.Provider
-        {
-            private AmazonS3Client client;
-            private final String bucket;
-            private final String key;
-            private boolean opened = false;
-
-            public SingleFileProvider(PluginTask task, int taskIndex)
-            {
-                this.client = newS3Client(task);
-                this.bucket = task.getBucket();
-                this.key = task.getFiles().get(taskIndex);
-            }
-
-            @Override
-            public InputStream openNext() throws IOException
-            {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-                GetObjectRequest request = new GetObjectRequest(bucket, key);
-                S3Object obj = client.getObject(request);
-                return new RetryableInputStream(obj.getObjectContent(), new S3RetryableOpener(client, request, obj.getObjectMetadata().getContentLength()));
-            }
-
-            @Override
-            public void close() { }
-        }
-
-        public S3FileInput(PluginTask task, int taskIndex)
-        {
-            super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
-        }
-
-        public void abort() { }
-
-        public CommitReport commit()
-        {
-            return Exec.newCommitReport();
-        }
-
-        @Override
-        public void close() { }
-    }
 }