embulk-input-s3 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f03ab171561f7cca3ff6fe8e17c63b73f31e2db3
-  data.tar.gz: a636f66a873c449741f09c51a7b415f550517d26
+  metadata.gz: db7314fde9364e1d4ec9edbb67f90cfc5c93dbe7
+  data.tar.gz: b3133c6d3ea81cef907d8cd974b5471186e086b6
 SHA512:
-  metadata.gz: 24baf759d3956a23317ed73f5878d9ec8d66bc493ba344131a16b325fb3558dd5fbab332dacfcac2f7abc89ecd3518cec89f4c232c7d17135159fa95fb444fac
-  data.tar.gz: c83e6f98d5d03e45f83641d879a80c41be41d292d999a1b90ab783ab02f296faa77a3a2276b447804e78dd2908a8b1f7cc026105688eff34fea0fdbbde92130b
+  metadata.gz: 0cd7e8ce269c322e7a03262267ae23370a9cff12877b9f33b8bebf1769ebdf093896babe8193324a3f4aa5b37ccae6a67b380e94367ab65fcc381b3d0e9a848e
+  data.tar.gz: 0e934e0baf997bfb29b3e3ed337daa28133a38a513050931902dc5f3d2cd03f652f6b2cb31e7d8fecd5b7e3dfccab869ede056053892af9e3bab2dc8cbd1ae4b

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.input.s3;
 import java.util.List;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Iterator;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
@@ -14,8 +15,6 @@ import com.google.common.base.Throwables;
 import org.slf4j.Logger;
 import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
-import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.ListObjectsRequest;
 import com.amazonaws.services.s3.model.S3ObjectSummary;
@@ -48,7 +47,7 @@ public abstract class AbstractS3FileInputPlugin
     private final Logger log = Exec.getLogger(S3FileInputPlugin.class);
 
     public interface PluginTask
-            extends Task
+            extends AwsCredentialsTask, FileList.Task, Task
     {
         @Config("bucket")
         public String getBucket();
@@ -64,16 +63,10 @@ public abstract class AbstractS3FileInputPlugin
         @ConfigDefault("null")
         public Optional<String> getAccessKeyId();
 
-        @Config("secret_access_key")
-        @ConfigDefault("null")
-        public Optional<String> getSecretAccessKey();
-
         // TODO timeout, ssl, etc
 
-        // TODO support more options such as STS
-
-        public List<String> getFiles();
-        public void setFiles(List<String> files);
+        public FileList getFiles();
+        public void setFiles(FileList files);
 
         @ConfigInject
         public BufferAllocator getBufferAllocator();
@@ -90,7 +83,7 @@ public abstract class AbstractS3FileInputPlugin
         task.setFiles(listFiles(task));
 
         // number of processors is same with number of files
-        return resume(task.dump(), task.getFiles().size(), control);
+        return resume(task.dump(), task.getFiles().getTaskCount(), control);
     }
 
     @Override
@@ -109,16 +102,7 @@ public abstract class AbstractS3FileInputPlugin
         ConfigDiff configDiff = Exec.newConfigDiff();
 
         // last_path
-        if (task.getFiles().isEmpty()) {
-            // keep the last value
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
-            }
-        } else {
-            List<String> files = new ArrayList<String>(task.getFiles());
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
-        }
+        configDiff.set("last_path", task.getFiles().getLastPath(task.getLastPath()));
 
         return configDiff;
     }
@@ -138,24 +122,7 @@ public abstract class AbstractS3FileInputPlugin
 
     protected AWSCredentialsProvider getCredentialsProvider(PluginTask task)
     {
-        final AWSCredentials cred;
-        if (task.getAccessKeyId().isPresent()) {
-            cred = new BasicAWSCredentials(
-                    task.getAccessKeyId().get(),
-                    task.getSecretAccessKey().get());
-        } else {
-            cred = new AnonymousAWSCredentials();
-        }
-        return new AWSCredentialsProvider() {
-            public AWSCredentials getCredentials()
-            {
-                return cred;
-            }
-
-            public void refresh()
-            {
-            }
-        };
+        return AwsCredentials.getAWSCredentialsProvider(task);
     }
 
     protected ClientConfiguration getClientConfiguration(PluginTask task)
@@ -170,7 +137,7 @@ public abstract class AbstractS3FileInputPlugin
         return clientConfig;
     }
 
-    private List<String> listFiles(PluginTask task)
+    private FileList listFiles(PluginTask task)
     {
         AmazonS3Client client = newS3Client(task);
         String bucketName = task.getBucket();
@@ -179,7 +146,10 @@ public abstract class AbstractS3FileInputPlugin
             log.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
         }
 
-        return listS3FilesByPrefix(client, bucketName, task.getPathPrefix(), task.getLastPath());
+        FileList.Builder builder = new FileList.Builder(task);
+        listS3FilesByPrefix(builder, client, bucketName,
+                task.getPathPrefix(), task.getLastPath());
+        return builder.build();
     }
 
     /**
@@ -187,24 +157,24 @@ public abstract class AbstractS3FileInputPlugin
      *
      * The resulting list does not include the file that's size == 0.
      */
-    public static List<String> listS3FilesByPrefix(AmazonS3Client client, String bucketName,
+    public static void listS3FilesByPrefix(FileList.Builder builder,
+            AmazonS3Client client, String bucketName,
             String prefix, Optional<String> lastPath)
     {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-
         String lastKey = lastPath.orNull();
         do {
             ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
             ObjectListing ol = client.listObjects(req);
-            for(S3ObjectSummary s : ol.getObjectSummaries()) {
+            for (S3ObjectSummary s : ol.getObjectSummaries()) {
                 if (s.getSize() > 0) {
-                    builder.add(s.getKey());
+                    builder.add(s.getKey(), s.getSize());
+                    if (!builder.needsMore()) {
+                        return;
+                    }
                 }
             }
             lastKey = ol.getNextMarker();
         } while(lastKey != null);
-
-        return builder.build();
     }
 
     @Override
@@ -308,24 +278,22 @@ public abstract class AbstractS3FileInputPlugin
     {
         private AmazonS3Client client;
         private final String bucket;
-        private final String key;
-        private boolean opened = false;
+        private final Iterator<String> iterator;
 
         public SingleFileProvider(PluginTask task, int taskIndex)
         {
             this.client = newS3Client(task);
             this.bucket = task.getBucket();
-            this.key = task.getFiles().get(taskIndex);
+            this.iterator = task.getFiles().get(taskIndex).iterator();
         }
 
         @Override
         public InputStream openNext() throws IOException
         {
-            if (opened) {
+            if (!iterator.hasNext()) {
                 return null;
             }
-            opened = true;
-            GetObjectRequest request = new GetObjectRequest(bucket, key);
+            GetObjectRequest request = new GetObjectRequest(bucket, iterator.next());
             S3Object obj = client.getObject(request);
             return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, obj.getObjectMetadata().getContentLength()));
         }

data/src/main/java/org/embulk/input/s3/AwsCredentials.java

@@ -0,0 +1,179 @@
+package org.embulk.input.s3;
+
+import com.google.common.base.Optional;
+import com.amazonaws.auth.AWSCredentials;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.auth.AWSSessionCredentials;
+import com.amazonaws.auth.AWSSessionCredentialsProvider;
+import com.amazonaws.auth.AnonymousAWSCredentials;
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
+import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
+import com.amazonaws.auth.InstanceProfileCredentialsProvider;
+import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
+import com.amazonaws.auth.profile.ProfileCredentialsProvider;
+import com.amazonaws.auth.profile.ProfilesConfigFile;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.unit.LocalFile;
+import org.slf4j.Logger;
+
+public abstract class AwsCredentials
+{
+    private AwsCredentials() { }
+
+    public static AWSCredentialsProvider getAWSCredentialsProvider(AwsCredentialsTask task)
+    {
+        switch (task.getAuthMethod()) {
+        case "basic":
+            // for backward compatibility
+            if (!task.getAccessKeyId().isPresent() && !task.getAccessKeyId().isPresent()) {
+                final Logger log = Exec.getLogger(AwsCredentials.class);
+                log.warn("Both access_key_id and secret_access_key are not set. Assuming that 'auth_method: anonymous' option is set.");
+                log.warn("If you intentionally use anonymous authentication, please set 'auth_method: anonymous' option.");
+                log.warn("This behavior will be removed in a futurte release.");
+                reject(task.getSessionToken(), "session_token");
+                reject(task.getProfileFile(), "profile_file");
+                reject(task.getProfileName(), "profile_name");
+                return new AWSCredentialsProvider() {
+                    public AWSCredentials getCredentials()
+                    {
+                        return new AnonymousAWSCredentials();
+                    }
+
+                    public void refresh() { }
+                };
+            }
+            else {
+                String accessKeyId = require(task.getAccessKeyId(), "'access_key_id', 'secret_access_key'");
+                String secretAccessKey = require(task.getSecretAccessKey(), "'secret_access_key'");
+                reject(task.getSessionToken(), "session_token");
+                reject(task.getProfileFile(), "profile_file");
+                reject(task.getProfileName(), "profile_name");
+                final BasicAWSCredentials creds = new BasicAWSCredentials(accessKeyId, secretAccessKey);
+                return new AWSCredentialsProvider() {
+                    public AWSCredentials getCredentials()
+                    {
+                        return creds;
+                    }
+
+                    public void refresh() { }
+                };
+            }
+
+        case "env":
+            reject(task.getAccessKeyId(), "access_key_id");
+            reject(task.getSecretAccessKey(), "secret_access_key");
+            reject(task.getSessionToken(), "session_token");
+            reject(task.getProfileFile(), "profile_file");
+            reject(task.getProfileName(), "profile_name");
+            return overwriteBasicCredentials(task, new EnvironmentVariableCredentialsProvider().getCredentials());
+
+        case "instance":
+            reject(task.getAccessKeyId(), "access_key_id");
+            reject(task.getSecretAccessKey(), "secret_access_key");
+            reject(task.getSessionToken(), "session_token");
+            reject(task.getProfileFile(), "profile_file");
+            reject(task.getProfileName(), "profile_name");
+            return new InstanceProfileCredentialsProvider();
+
+        case "profile":
+            {
+                reject(task.getAccessKeyId(), "access_key_id");
+                reject(task.getSecretAccessKey(), "secret_access_key");
+                reject(task.getSessionToken(), "session_token");
+
+                String profileName = task.getProfileName().or("default");
+                ProfileCredentialsProvider provider;
+                if (task.getProfileFile().isPresent()) {
+                    ProfilesConfigFile file = new ProfilesConfigFile(task.getProfileFile().get().getFile());
+                    provider = new ProfileCredentialsProvider(file, profileName);
+                }
+                else {
+                    provider = new ProfileCredentialsProvider(profileName);
+                }
+                task.setProfileName(Optional.<String>absent());
+                task.setProfileFile(Optional.<LocalFile>absent());
+
+                return overwriteBasicCredentials(task, provider.getCredentials());
+            }
+
+        case "properties":
+            reject(task.getAccessKeyId(), "access_key_id");
+            reject(task.getSecretAccessKey(), "secret_access_key");
+            reject(task.getSessionToken(), "session_token");
+            reject(task.getProfileFile(), "profile_file");
+            reject(task.getProfileName(), "profile_name");
+            return overwriteBasicCredentials(task, new SystemPropertiesCredentialsProvider().getCredentials());
+
+        case "anonymous":
+            reject(task.getAccessKeyId(), "access_key_id");
+            reject(task.getSecretAccessKey(), "secret_access_key");
+            reject(task.getSessionToken(), "session_token");
+            reject(task.getProfileFile(), "profile_file");
+            reject(task.getProfileName(), "profile_name");
+            return new AWSCredentialsProvider() {
+                public AWSCredentials getCredentials()
+                {
+                    return new AnonymousAWSCredentials();
+                }
+
+                public void refresh() { }
+            };
+
+        case "session":
+            {
+                String accessKeyId = require(task.getAccessKeyId(), "'access_key_id', 'secret_access_key', 'session_token'");
+                String secretAccessKey = require(task.getSecretAccessKey(), "'secret_access_key', 'session_token'");
+                String sessionToken = require(task.getSessionToken(), "'session_token'");
+                reject(task.getProfileFile(), "profile_file");
+                reject(task.getProfileName(), "profile_name");
+                final AWSSessionCredentials creds = new BasicSessionCredentials(accessKeyId, secretAccessKey, sessionToken);
+                return new AWSSessionCredentialsProvider() {
+                    public AWSSessionCredentials getCredentials()
+                    {
+                        return creds;
+                    }
+
+                    public void refresh() { }
+                };
+            }
+
+        default:
+            throw new ConfigException(String.format("Unknwon auth_method '%s'. Supported methods are basic, instance, profile, properties, anonymous, and session.",
+                        task.getAuthMethod()));
+        }
+    }
+
+    private static AWSCredentialsProvider overwriteBasicCredentials(AwsCredentialsTask task, final AWSCredentials creds)
+    {
+        task.setAuthMethod("basic");
+        task.setAccessKeyId(Optional.of(creds.getAWSAccessKeyId()));
+        task.setSecretAccessKey(Optional.of(creds.getAWSSecretKey()));
+        return new AWSCredentialsProvider() {
+            public AWSCredentials getCredentials()
+            {
+                return creds;
+            }
+
+            public void refresh() { }
+        };
+    }
+
+    private static <T> T require(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            return value.get();
+        }
+        else {
+            throw new ConfigException("Required option is not set: " + message);
+        }
+    }
+
+    private static <T> void reject(Optional<T> value, String message)
+    {
+        if (value.isPresent()) {
+            throw new ConfigException("Invalid option is set: " + message);
+        }
+    }
+}

data/src/main/java/org/embulk/input/s3/AwsCredentialsTask.java

@@ -0,0 +1,39 @@
+package org.embulk.input.s3;
+
+import com.google.common.base.Optional;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.spi.unit.LocalFile;
+
+public interface AwsCredentialsTask
+{
+    @Config("auth_method")
+    @ConfigDefault("\"basic\"")
+    String getAuthMethod();
+    void setAuthMethod(String method);
+
+    @Config("access_key_id")
+    @ConfigDefault("null")
+    Optional<String> getAccessKeyId();
+    void setAccessKeyId(Optional<String> value);
+
+    @Config("secret_access_key")
+    @ConfigDefault("null")
+    Optional<String> getSecretAccessKey();
+    void setSecretAccessKey(Optional<String> value);
+
+    @Config("session_token")
+    @ConfigDefault("null")
+    Optional<String> getSessionToken();
+    void setSessionToken(Optional<String> value);
+
+    @Config("profile_file")
+    @ConfigDefault("null")
+    Optional<LocalFile> getProfileFile();
+    void setProfileFile(Optional<LocalFile> value);
+
+    @Config("profile_name")
+    @ConfigDefault("null")
+    Optional<String> getProfileName();
+    void setProfileName(Optional<String> value);
+}

data/src/main/java/org/embulk/input/s3/FileList.java

@@ -0,0 +1,289 @@
+package org.embulk.input.s3;
+
+import java.util.List;
+import java.util.AbstractList;
+import java.util.ArrayList;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedInputStream;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigSource;
+import com.google.common.base.Throwables;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonCreator;
+
+// this class should be moved to embulk-core
+public class FileList
+{
+    public interface Task
+    {
+        @Config("total_file_count_limit")
+        @ConfigDefault("2147483647")
+        int getTotalFileCountLimit();
+    }
+
+    public static class Entry
+    {
+        private int index;
+        private long size;
+
+        @JsonCreator
+        public Entry(
+                @JsonProperty("index") int index,
+                @JsonProperty("size") long size)
+        {
+            this.index = index;
+            this.size = size;
+        }
+
+        @JsonProperty("index")
+        public int getIndex() { return index; }
+
+        @JsonProperty("size")
+        public long getSize() { return size; }
+    }
+
+    public static class Builder
+    {
+        private final ByteArrayOutputStream binary;
+        private final OutputStream stream;
+        private final List<Entry> entries = new ArrayList<>();
+        private String last = null;
+
+        private int limitCount = Integer.MAX_VALUE;
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public Builder(Task task)
+        {
+            this();
+            this.limitCount = task.getTotalFileCountLimit();
+        }
+
+        public Builder(ConfigSource config)
+        {
+            this();
+            this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
+        }
+
+        public Builder()
+        {
+            binary = new ByteArrayOutputStream();
+            try {
+                stream = new BufferedOutputStream(new GZIPOutputStream(binary));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        public Builder limitTotalFileCount(int limitCount)
+        {
+            this.limitCount = limitCount;
+            return this;
+        }
+
+        public int size()
+        {
+            return entries.size();
+        }
+
+        public boolean needsMore()
+        {
+            return size() < limitCount;
+        }
+
+        public synchronized boolean add(String path, long size)
+        {
+            // TODO throw IllegalStateException if stream is already closed
+
+            if (!needsMore()) {
+                return false;
+            }
+
+            // TODO in the future, support some other filtering parameters (file name suffix filter, regex filter, etc)
+            //      and return false if filtered out.
+
+            int index = entries.size();
+            entries.add(new Entry(index, size));
+
+            byte[] data = path.getBytes(StandardCharsets.UTF_8);
+            castBuffer.putInt(0, data.length);
+            try {
+                stream.write(castBuffer.array());
+                stream.write(data);
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+
+            last = path;
+            return true;
+        }
+
+        public FileList build()
+        {
+            try {
+                stream.close();
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            return new FileList(binary.toByteArray(), getSplits(entries), Optional.fromNullable(last));
+        }
+
+        private List<List<Entry>> getSplits(List<Entry> all)
+        {
+            // TODO combine multiple entries into one task using some configuration parameters
+            List<List<Entry>> tasks = new ArrayList<>();
+            for (Entry entry : all) {
+                tasks.add(ImmutableList.of(entry));
+            }
+            return tasks;
+        }
+    }
+
+    private final byte[] data;
+    private final List<List<Entry>> tasks;
+    private final Optional<String> last;
+
+    @JsonCreator
+    @Deprecated
+    public FileList(
+            @JsonProperty("data") byte[] data,
+            @JsonProperty("tasks") List<List<Entry>> tasks,
+            @JsonProperty("last") Optional<String> last)
+    {
+        this.data = data;
+        this.tasks = tasks;
+        this.last = last;
+    }
+
+    @JsonIgnore
+    public Optional<String> getLastPath(Optional<String> lastLastPath)
+    {
+        if (last.isPresent()) {
+            return last;
+        }
+        return lastLastPath;
+    }
+
+    @JsonIgnore
+    public int getTaskCount()
+    {
+        return tasks.size();
+    }
+
+    @JsonIgnore
+    public List<String> get(int i)
+    {
+        return new EntryList(data, tasks.get(i));
+    }
+
+    @JsonProperty("data")
+    @Deprecated
+    public byte[] getData()
+    {
+        return data;
+    }
+
+    @JsonProperty("tasks")
+    @Deprecated
+    public List<List<Entry>> getTasks()
+    {
+        return tasks;
+    }
+
+    @JsonProperty("last")
+    @Deprecated
+    public Optional<String> getLast()
+    {
+        return last;
+    }
+
+    private class EntryList
+            extends AbstractList<String>
+    {
+        private final byte[] data;
+        private final List<Entry> entries;
+        private InputStream stream;
+        private int current;
+
+        private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
+
+        public EntryList(byte[] data, List<Entry> entries)
+        {
+            this.data = data;
+            this.entries = entries;
+            try {
+                this.stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.current = 0;
+        }
+
+        @Override
+        public synchronized String get(int i)
+        {
+            Entry e = entries.get(i);
+            if (e.getIndex() < current) {
+                // rewind to the head
+                try {
+                    stream.close();
+                    stream = new BufferedInputStream(new GZIPInputStream(new ByteArrayInputStream(data)));
+                }
+                catch (IOException ex) {
+                    throw Throwables.propagate(ex);
+                }
+                current = 0;
+            }
+
+            while (current < e.getIndex()) {
+                readNext();
+            }
+            // now current == e.getIndex()
+            return readNextString();
+        }
+
+        @Override
+        public int size()
+        {
+            return entries.size();
+        }
+
+        private byte[] readNext()
+        {
+            try {
+                stream.read(castBuffer.array());
+                int n = castBuffer.getInt(0);
+                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                stream.read(b);
+
+                current++;
+
+                return b;
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        private String readNextString()
+        {
+            return new String(readNext(), StandardCharsets.UTF_8);
+        }
+    }
+}

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -72,7 +72,9 @@ public class TestS3FileInputPlugin
         doReturn("in/file/").doReturn(null).when(ol).getNextMarker();
 
         // It counts only size != 0 files.
-        assertEquals(1, S3FileInputPlugin.listS3FilesByPrefix(client, "bucketName", "prefix", Optional.<String>absent()).size());
+        FileList.Builder builder = new FileList.Builder();
+        S3FileInputPlugin.listS3FilesByPrefix(builder, client, "bucketName", "prefix", Optional.<String>absent());
+        assertEquals(1, builder.size());
     }
 
     @Test
@@ -90,7 +92,7 @@ public class TestS3FileInputPlugin
                 public List<TaskReport> run(TaskSource taskSource, int taskCount)
                 {
                     assertEquals(3, taskCount);
-                    List<String> files = taskSource.loadTask(S3PluginTask.class).getFiles();
+                    List<String> files = fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles());
                     assertArrayEquals(new String[]{"in/aa/a", "in/aa/b", "in/aa/c"}, files.toArray(new String[files.size()]));
                     return emptyTaskReports(taskCount);
                 }
@@ -108,12 +110,12 @@ public class TestS3FileInputPlugin
                 public List<TaskReport> run(TaskSource taskSource, int taskCount)
                 {
                     assertEquals(0, taskCount);
-                    assertTrue(taskSource.loadTask(S3PluginTask.class).getFiles().isEmpty());
+                    assertTrue(fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles()).isEmpty());
                     return emptyTaskReports(taskCount);
                 }
             });
 
-            assertFalse(configDiff.has("last_path"));
+            assertEquals(null, configDiff.get(String.class, "last_path", null));
         }
 
         { // if files are empty, keep the previous last_path.
@@ -126,7 +128,7 @@ public class TestS3FileInputPlugin
                 @Override
                 public List<TaskReport> run(TaskSource taskSource, int taskCount) {
                     assertEquals(0, taskCount);
-                    assertTrue(taskSource.loadTask(S3PluginTask.class).getFiles().isEmpty());
+                    assertTrue(fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles()).isEmpty());
                     return emptyTaskReports(taskCount);
                 }
             });
@@ -143,7 +145,9 @@ public class TestS3FileInputPlugin
         doReturn(client).when(plugin).newS3Client(any(PluginTask.class));
 
         PluginTask task = config.loadConfig(plugin.getTaskClass());
-        task.setFiles(Arrays.asList(new String[]{"in/aa/a"}));
+        FileList.Builder builder = new FileList.Builder();
+        builder.add("in/aa/a", 100);
+        task.setFiles(builder.build());
 
         StringBuilder sbuf = new StringBuilder();
         try (S3FileInput input = (S3FileInput) plugin.open(task.dump(), 0)) {
@@ -218,4 +222,15 @@ public class TestS3FileInputPlugin
         }
         return reports.build();
     }
+
+    private static List<String> fileListToList(FileList list)
+    {
+        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        for (int i=0; i < list.getTaskCount(); i++) {
+            for (String path : list.get(i)) {
+                builder.add(path);
+            }
+        }
+        return builder.build();
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-15 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -48,18 +48,20 @@ files:
 - build.gradle
 - lib/embulk/input/s3.rb
 - src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
+- src/main/java/org/embulk/input/s3/AwsCredentials.java
+- src/main/java/org/embulk/input/s3/AwsCredentialsTask.java
+- src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
-- classpath/aws-java-sdk-core-1.9.22.jar
-- classpath/aws-java-sdk-kms-1.9.22.jar
-- classpath/aws-java-sdk-s3-1.9.22.jar
+- classpath/aws-java-sdk-core-1.10.33.jar
+- classpath/aws-java-sdk-kms-1.10.33.jar
+- classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.3.jar
-- classpath/httpclient-4.3.4.jar
-- classpath/httpcore-4.3.2.jar
+- classpath/embulk-input-s3-0.2.4.jar
+- classpath/httpclient-4.3.6.jar
+- classpath/httpcore-4.3.3.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
-- classpath/joda-time-2.8.2.jar
 homepage: https://github.com/embulk/embulk-input-s3
 licenses:
 - Apache 2.0