embulk-input-s3 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: db7314fde9364e1d4ec9edbb67f90cfc5c93dbe7
-  data.tar.gz: b3133c6d3ea81cef907d8cd974b5471186e086b6
+  metadata.gz: 857c5f811620e1273a6fc5ec69a4eeefb658f1b0
+  data.tar.gz: deb852435fb0cee1d2db0391cd3a8d0f30255638
 SHA512:
-  metadata.gz: 0cd7e8ce269c322e7a03262267ae23370a9cff12877b9f33b8bebf1769ebdf093896babe8193324a3f4aa5b37ccae6a67b380e94367ab65fcc381b3d0e9a848e
-  data.tar.gz: 0e934e0baf997bfb29b3e3ed337daa28133a38a513050931902dc5f3d2cd03f652f6b2cb31e7d8fecd5b7e3dfccab869ede056053892af9e3bab2dc8cbd1ae4b
+  metadata.gz: f5dd533e7e7d5fd366268496118c110689bde80d4f5c98469eb9ff1be77734a168080a24f52d3415dcda158ce5cd1ace35c9025fc4b654663c9726f3994b9bc3
+  data.tar.gz: 1d5cab1c9c166a6352dbab3a30e65d058740e3b1883bb0776fac7bc6b677b6fe2bc8e047dba6e006f75cae722e0416504e8d6f493c6466dd7d54be3e484c79a7

data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java

@@ -22,6 +22,7 @@ import com.amazonaws.services.s3.model.ObjectListing;
 import com.amazonaws.services.s3.model.GetObjectRequest;
 import com.amazonaws.services.s3.model.S3Object;
 import com.amazonaws.ClientConfiguration;
+import com.amazonaws.AmazonServiceException;
 import com.amazonaws.Protocol;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigInject;
@@ -31,6 +32,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.TaskReport;
+import org.embulk.config.ConfigException;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -139,17 +141,29 @@ public abstract class AbstractS3FileInputPlugin
 
     private FileList listFiles(PluginTask task)
     {
-        AmazonS3Client client = newS3Client(task);
-        String bucketName = task.getBucket();
+        try {
+            AmazonS3Client client = newS3Client(task);
+            String bucketName = task.getBucket();
 
-        if (task.getPathPrefix().equals("/")) {
-            log.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
-        }
+            if (task.getPathPrefix().equals("/")) {
+                log.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
+            }
 
-        FileList.Builder builder = new FileList.Builder(task);
-        listS3FilesByPrefix(builder, client, bucketName,
-                task.getPathPrefix(), task.getLastPath());
-        return builder.build();
+            FileList.Builder builder = new FileList.Builder(task);
+            listS3FilesByPrefix(builder, client, bucketName,
+                    task.getPathPrefix(), task.getLastPath());
+            return builder.build();
+        }
+        catch (AmazonServiceException ex) {
+            if (ex.getErrorType().equals(AmazonServiceException.ErrorType.Client)) {
+                // HTTP 40x errors. auth error, bucket doesn't exist, etc. See AWS document for the full list:
+                // http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html
+                if (ex.getStatusCode() != 400) {  // 404 Bad Request is unexpected error
+                    throw new ConfigException(ex);
+                }
+            }
+            throw ex;
+        }
     }
 
     /**

data/src/main/java/org/embulk/input/s3/FileList.java

@@ -5,6 +5,8 @@ import java.util.AbstractList;
 import java.util.ArrayList;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.BufferedOutputStream;
@@ -29,6 +31,10 @@ public class FileList
 {
     public interface Task
     {
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
         @Config("total_file_count_limit")
         @ConfigDefault("2147483647")
         int getTotalFileCountLimit();
@@ -63,17 +69,21 @@ public class FileList
         private String last = null;
 
         private int limitCount = Integer.MAX_VALUE;
+        private Pattern pathMatchPattern;
+
         private final ByteBuffer castBuffer = ByteBuffer.allocate(4);
 
         public Builder(Task task)
         {
             this();
             this.limitCount = task.getTotalFileCountLimit();
+            this.pathMatchPattern = Pattern.compile(task.getPathMatchPattern());
         }
 
         public Builder(ConfigSource config)
         {
             this();
+            this.pathMatchPattern = Pattern.compile(config.get(String.class, "path_match_pattern", ".*"));
             this.limitCount = config.get(int.class, "total_file_count_limit", Integer.MAX_VALUE);
         }
 
@@ -94,6 +104,12 @@ public class FileList
             return this;
         }
 
+        public Builder pathMatchPattern(String pattern)
+        {
+            this.pathMatchPattern = Pattern.compile(pattern);
+            return this;
+        }
+
         public int size()
         {
             return entries.size();
@@ -104,6 +120,7 @@ public class FileList
             return size() < limitCount;
         }
 
+        // returns true if this file is used
         public synchronized boolean add(String path, long size)
         {
             // TODO throw IllegalStateException if stream is already closed
@@ -112,8 +129,9 @@ public class FileList
                 return false;
             }
 
-            // TODO in the future, support some other filtering parameters (file name suffix filter, regex filter, etc)
-            //      and return false if filtered out.
+            if (!pathMatchPattern.matcher(path).matches()) {
+                return false;
+            }
 
             int index = entries.size();
             entries.add(new Entry(index, size));

data/src/test/java/org/embulk/input/s3/TestAwsCredentials.java

@@ -0,0 +1,173 @@
+package org.embulk.input.s3;
+
+import com.amazonaws.auth.BasicAWSCredentials;
+import com.amazonaws.auth.BasicSessionCredentials;
+import com.amazonaws.auth.policy.Policy;
+import com.amazonaws.auth.policy.Resource;
+import com.amazonaws.auth.policy.Statement;
+import com.amazonaws.auth.policy.actions.S3Actions;
+import com.amazonaws.internal.StaticCredentialsProvider;
+import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
+import com.amazonaws.services.securitytoken.model.Credentials;
+import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
+import com.amazonaws.services.securitytoken.model.GetFederationTokenResult;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.s3.TestS3FileInputPlugin.Control;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.TestPageBuilderReader;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.embulk.input.s3.TestS3FileInputPlugin.assertRecords;
+import static org.embulk.input.s3.TestS3FileInputPlugin.parserConfig;
+import static org.embulk.input.s3.TestS3FileInputPlugin.schemaConfig;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+public class TestAwsCredentials
+{
+    private static String EMBULK_S3_TEST_BUCKET;
+    private static String EMBULK_S3_TEST_ACCESS_KEY_ID;
+    private static String EMBULK_S3_TEST_SECRET_ACCESS_KEY;
+    private static final String EMBULK_S3_TEST_PATH_PREFIX = "embulk_input_s3_test";
+
+    /*
+     * This test case requires environment variables:
+     *   EMBULK_S3_TEST_BUCKET
+     *   EMBULK_S3_TEST_ACCESS_KEY_ID
+     *   EMBULK_S3_TEST_SECRET_ACCESS_KEY
+     * If the variables not set, the test case is skipped.
+     */
+    @BeforeClass
+    public static void initializeConstantVariables()
+    {
+        EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET");
+        EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID");
+        EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY");
+        assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY);
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    private ConfigSource config;
+    private FileInputRunner runner;
+    private TestPageBuilderReader.MockPageOutput output;
+
+    @Before
+    public void createResources()
+    {
+        config = runtime.getExec().newConfigSource()
+                .set("type", "s3")
+                .set("bucket", EMBULK_S3_TEST_BUCKET)
+                .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX)
+                .set("parser", parserConfig(schemaConfig()));
+        runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class));
+        output = new TestPageBuilderReader.MockPageOutput();
+    }
+
+    private void doTest(ConfigSource config)
+    {
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+
+        assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path"));
+        assertRecords(config, output);
+    }
+
+    @Test
+    public void useBasic()
+    {
+        ConfigSource config = this.config.deepCopy()
+                .set("auth_method", "basic")
+                .set("access_key_id", EMBULK_S3_TEST_ACCESS_KEY_ID)
+                .set("secret_access_key", EMBULK_S3_TEST_SECRET_ACCESS_KEY);
+        doTest(config);
+    }
+
+    @Test
+    public void useEnv()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useInstance()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useProfile()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useProperties()
+    {
+        String origAccessKeyId = System.getProperty("aws.accessKeyId");
+        String origSecretKey = System.getProperty("aws.secretKey");
+        try {
+
+            ConfigSource config = this.config.deepCopy().set("auth_method", "properties");
+            System.setProperty("aws.accessKeyId", EMBULK_S3_TEST_ACCESS_KEY_ID);
+            System.setProperty("aws.secretKey", EMBULK_S3_TEST_SECRET_ACCESS_KEY);
+            doTest(config);
+        }
+        finally {
+            if (origAccessKeyId != null) {
+                System.setProperty("aws.accessKeyId", origAccessKeyId);
+            }
+            if (origSecretKey != null) {
+                System.setProperty("aws.secretKey", origAccessKeyId);
+            }
+        }
+    }
+
+    @Test
+    public void useAnonymous()
+    {
+        // TODO
+    }
+
+    @Test
+    public void useSession()
+    {
+        BasicSessionCredentials sessionCredentials = getSessionCredentials();
+        ConfigSource config = this.config.deepCopy()
+                .set("auth_method", "session")
+                .set("access_key_id", sessionCredentials.getAWSAccessKeyId())
+                .set("secret_access_key", sessionCredentials.getAWSSecretKey())
+                .set("session_token", sessionCredentials.getSessionToken());
+        doTest(config);
+    }
+
+    private static BasicSessionCredentials getSessionCredentials()
+    {
+        AWSSecurityTokenServiceClient stsClient = new AWSSecurityTokenServiceClient(
+                new StaticCredentialsProvider(new BasicAWSCredentials(EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY)));
+
+        GetFederationTokenRequest getFederationTokenRequest = new GetFederationTokenRequest();
+        getFederationTokenRequest.setDurationSeconds(7200);
+        getFederationTokenRequest.setName("dummy");
+
+        Policy policy = new Policy().withStatements(new Statement(Statement.Effect.Allow)
+                .withActions(S3Actions.ListObjects, S3Actions.GetObject)
+                .withResources(
+                        new Resource("arn:aws:s3:::" + EMBULK_S3_TEST_BUCKET + "/" + EMBULK_S3_TEST_PATH_PREFIX + "/*"),
+                        new Resource("arn:aws:s3:::" + EMBULK_S3_TEST_BUCKET)));
+        getFederationTokenRequest.setPolicy(policy.toJson());
+
+        GetFederationTokenResult federationTokenResult = stsClient.getFederationToken(getFederationTokenRequest);
+        Credentials sessionCredentials = federationTokenResult.getCredentials();
+
+        return new BasicSessionCredentials(
+                sessionCredentials.getAccessKeyId(),
+                sessionCredentials.getSecretAccessKey(),
+                sessionCredentials.getSessionToken());
+    }
+}

data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java

@@ -1,236 +1,220 @@
 package org.embulk.input.s3;
 
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.model.GetObjectRequest;
-import com.amazonaws.services.s3.model.ListObjectsRequest;
-import com.amazonaws.services.s3.model.ObjectListing;
-import com.amazonaws.services.s3.model.ObjectMetadata;
-import com.amazonaws.services.s3.model.S3Object;
-import com.amazonaws.services.s3.model.S3ObjectInputStream;
-import com.amazonaws.services.s3.model.S3ObjectSummary;
-import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
-import org.embulk.input.s3.AbstractS3FileInputPlugin.PluginTask;
-import org.embulk.input.s3.AbstractS3FileInputPlugin.S3FileInput;
-import org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.util.LineDecoder;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.PageOutput;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.Pages;
+import org.embulk.standards.CsvParserPlugin;
 import org.junit.Before;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.ByteArrayInputStream;
-import java.util.Arrays;
+import java.util.ArrayList;
 import java.util.List;
 
-import static org.junit.Assert.*;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assume.assumeNotNull;
 
 public class TestS3FileInputPlugin
 {
+    private static String EMBULK_S3_TEST_BUCKET;
+    private static String EMBULK_S3_TEST_ACCESS_KEY_ID;
+    private static String EMBULK_S3_TEST_SECRET_ACCESS_KEY;
+    private static final String EMBULK_S3_TEST_PATH_PREFIX = "embulk_input_s3_test";
+
+    /*
+     * This test case requires environment variables:
+     *   EMBULK_S3_TEST_BUCKET
+     *   EMBULK_S3_TEST_ACCESS_KEY_ID
+     *   EMBULK_S3_TEST_SECRET_ACCESS_KEY
+     * If the variables not set, the test case is skipped.
+     */
+    @BeforeClass
+    public static void initializeConstantVariables()
+    {
+        EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET");
+        EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID");
+        EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY");
+        assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY);
+    }
+
     @Rule
     public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
 
     private ConfigSource config;
-    private S3FileInputPlugin plugin;
-    private AmazonS3Client client;
+    private FileInputRunner runner;
+    private MockPageOutput output;
 
     @Before
     public void createResources()
     {
-        config = config();
-        plugin = spy(runtime.getInstance(S3FileInputPlugin.class));
-        client = mock(AmazonS3Client.class);
+        config = runtime.getExec().newConfigSource()
+                .set("type", "s3")
+                .set("bucket", EMBULK_S3_TEST_BUCKET)
+                .set("access_key_id", EMBULK_S3_TEST_ACCESS_KEY_ID)
+                .set("secret_access_key", EMBULK_S3_TEST_SECRET_ACCESS_KEY)
+                .set("path_prefix", EMBULK_S3_TEST_PATH_PREFIX)
+                .set("parser", parserConfig(schemaConfig()));
+        runner = new FileInputRunner(runtime.getInstance(S3FileInputPlugin.class));
+        output = new MockPageOutput();
     }
 
     @Test
-    public void checkS3ClientCreatedSuccessfully()
+    public void simpleTest()
     {
-        PluginTask task = config().loadConfig(plugin.getTaskClass());
-        plugin.newS3Client(task);
+        ConfigSource config = this.config.deepCopy();
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+
+        assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path"));
+        assertRecords(config, output);
     }
 
     @Test
-    public void listS3FilesByPrefix()
+    public void useLastPath()
+            throws Exception
     {
-        // AWSS3Client returns list1 for the first iteration and list2 next.
-        List<S3ObjectSummary> list1 = s3objects("in/", 0L, "in/file/", 0L, "in/file/sample.csv.gz", 12345L);
-        List<S3ObjectSummary> list2 = s3objects("sample2.csv.gz", 0L);
-        ObjectListing ol = mock(ObjectListing.class);
-
-        doReturn(list1).doReturn(list2).when(ol).getObjectSummaries();
-        doReturn(ol).when(client).listObjects(any(ListObjectsRequest.class));
-        doReturn("in/file/").doReturn(null).when(ol).getNextMarker();
-
-        // It counts only size != 0 files.
-        FileList.Builder builder = new FileList.Builder();
-        S3FileInputPlugin.listS3FilesByPrefix(builder, client, "bucketName", "prefix", Optional.<String>absent());
-        assertEquals(1, builder.size());
+        ConfigSource config = this.config.deepCopy().set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv");
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
+
+        assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path"));
+        assertEquals(0, getRecords(config, output).size());
     }
 
     @Test
-    public void checkLastPath()
+    public void emptyFilesWithLastPath()
+            throws Exception
     {
-        doReturn(null).when(client).listObjects(any(ListObjectsRequest.class));
-        doReturn(client).when(plugin).newS3Client(any(PluginTask.class));
-
-        { // set a last file to last_path
-            ObjectListing listing = listing("in/aa", 0L, "in/aa/a", 3L, "in/aa/b", 2L, "in/aa/c", 1L);
-            doReturn(listing).when(client).listObjects(any(ListObjectsRequest.class));
-
-            ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
-                @Override
-                public List<TaskReport> run(TaskSource taskSource, int taskCount)
-                {
-                    assertEquals(3, taskCount);
-                    List<String> files = fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles());
-                    assertArrayEquals(new String[]{"in/aa/a", "in/aa/b", "in/aa/c"}, files.toArray(new String[files.size()]));
-                    return emptyTaskReports(taskCount);
-                }
-            });
-
-            assertEquals("in/aa/c", configDiff.get(String.class, "last_path"));
-        }
+        ConfigSource config = this.config.deepCopy()
+                .set("path_prefix", "empty_files_prefix")
+                .set("last_path", EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv");
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
 
-        { // if files are empty and last_path is not specified, last_path is not set.
-            ObjectListing listing = listing("in/aa", 0L);
-            doReturn(listing).when(client).listObjects(any(ListObjectsRequest.class));
-
-            ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
-                @Override
-                public List<TaskReport> run(TaskSource taskSource, int taskCount)
-                {
-                    assertEquals(0, taskCount);
-                    assertTrue(fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles()).isEmpty());
-                    return emptyTaskReports(taskCount);
-                }
-            });
-
-            assertEquals(null, configDiff.get(String.class, "last_path", null));
-        }
-
-        { // if files are empty, keep the previous last_path.
-            config.set("last_path", "in/bb");
-
-            ObjectListing listing = listing("in/aa", 0L);
-            doReturn(listing).when(client).listObjects(any(ListObjectsRequest.class));
-
-            ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
-                @Override
-                public List<TaskReport> run(TaskSource taskSource, int taskCount) {
-                    assertEquals(0, taskCount);
-                    assertTrue(fileListToList(taskSource.loadTask(S3PluginTask.class).getFiles()).isEmpty());
-                    return emptyTaskReports(taskCount);
-                }
-            });
-
-            assertEquals("in/bb", configDiff.get(String.class, "last_path"));
-        }
+        assertEquals(EMBULK_S3_TEST_PATH_PREFIX + "/sample_01.csv", configDiff.get(String.class, "last_path")); // keep the last_path
+        assertEquals(0, getRecords(config, output).size());
     }
 
     @Test
-    public void checkS3FileInputByOpen()
+    public void useTotalFileCountLimit()
             throws Exception
     {
-        doReturn(s3object("in/aa/a", "aa")).when(client).getObject(any(GetObjectRequest.class));
-        doReturn(client).when(plugin).newS3Client(any(PluginTask.class));
-
-        PluginTask task = config.loadConfig(plugin.getTaskClass());
-        FileList.Builder builder = new FileList.Builder();
-        builder.add("in/aa/a", 100);
-        task.setFiles(builder.build());
-
-        StringBuilder sbuf = new StringBuilder();
-        try (S3FileInput input = (S3FileInput) plugin.open(task.dump(), 0)) {
-            LineDecoder d = new LineDecoder(input, config.loadConfig(LineDecoder.DecoderTask.class));
-            while (d.nextFile()) {
-                sbuf.append(d.poll());
-            }
-        }
-        assertEquals("aa", sbuf.toString());
-    }
+        ConfigSource config = this.config.deepCopy().set("total_file_count_limit", 0);
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
 
-    public static ConfigSource config()
-    {
-        return Exec.newConfigSource()
-                .set("bucket", "my_bucket")
-                .set("path_prefix", "my_path_prefix")
-                .set("access_key_id", "my_access_key_id")
-                .set("secret_access_key", "my_secret_access_key");
+        assertNull(configDiff.get(String.class, "last_path"));
+        assertEquals(0, getRecords(config, output).size());
     }
 
-    static ObjectListing listing(Object... keySizes)
+    @Test
+    public void usePathMatchPattern()
+            throws Exception
     {
-        ObjectListing listing = mock(ObjectListing.class);
-        if (keySizes == null) {
-            return listing;
-        }
+        ConfigSource config = this.config.deepCopy().set("path_match_pattern", "/match/");
+        ConfigDiff configDiff = runner.transaction(config, new Control(runner, output));
 
-        List<S3ObjectSummary> s3objects = s3objects(keySizes);
-        doReturn(s3objects).when(listing).getObjectSummaries();
-        doReturn(null).when(listing).getNextMarker();
-        return listing;
+        assertNull(configDiff.get(String.class, "last_path"));
+        assertEquals(0, getRecords(config, output).size());
     }
 
-    static List<S3ObjectSummary> s3objects(Object... keySizes)
+    static class Control
+            implements InputPlugin.Control
     {
-        ImmutableList.Builder<S3ObjectSummary> builder = new ImmutableList.Builder<>();
-        if (keySizes == null) {
-            return builder.build();
+        private FileInputRunner runner;
+        private PageOutput output;
+
+        Control(FileInputRunner runner, PageOutput output)
+        {
+            this.runner = runner;
+            this.output = output;
         }
 
-        for (int i = 0; i < keySizes.length; i += 2) {
-            String key = (String) keySizes[i];
-            long size = (Long) keySizes[i + 1];
-            builder.add(s3object(key, size));
+        @Override
+        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(runner.run(taskSource, schema, i, output));
+            }
+            return reports;
         }
-        return builder.build();
     }
 
-    static S3ObjectSummary s3object(String key, long size)
+    static ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
     {
-        S3ObjectSummary o = new S3ObjectSummary();
-        o.setKey(key);
-        o.setSize(size);
-        return o;
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 0);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
     }
 
-    static S3Object s3object(String key, String value)
+    static ImmutableList<Object> schemaConfig()
     {
-        S3Object o = new S3Object();
-        o.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(value.getBytes()), null));
-        ObjectMetadata om = new ObjectMetadata();
-        om.setContentLength(value.length());
-        o.setObjectMetadata(om);
-        return o;
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "timestamp", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "host", "type", "string"));
+        builder.add(ImmutableMap.of("name", "path", "type", "string"));
+        builder.add(ImmutableMap.of("name", "method", "type", "string"));
+        builder.add(ImmutableMap.of("name", "referer", "type", "string"));
+        builder.add(ImmutableMap.of("name", "code", "type", "long"));
+        builder.add(ImmutableMap.of("name", "agent", "type", "string"));
+        builder.add(ImmutableMap.of("name", "user", "type", "string"));
+        builder.add(ImmutableMap.of("name", "size", "type", "long"));
+        return builder.build();
     }
 
-    static List<TaskReport> emptyTaskReports(int taskCount)
+    static void assertRecords(ConfigSource config, MockPageOutput output)
     {
-        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
-        for (int i = 0; i < taskCount; i++) {
-            reports.add(Exec.newTaskReport());
+        List<Object[]> records = getRecords(config, output);
+
+        assertEquals(2, records.size());
+        {
+            Object[] record = records.get(0);
+            assertEquals("2014-10-02 22:15:39 UTC", record[0].toString());
+            assertEquals("84.186.29.187", record[1]);
+            assertEquals("/category/electronics", record[2]);
+            assertEquals("GET", record[3]);
+            assertEquals("/category/music", record[4]);
+            assertEquals(200L, record[5]);
+            assertEquals("Mozilla/5.0", record[6]);
+            assertEquals("-", record[7]);
+            assertEquals(136L, record[8]);
+        }
+
+        {
+            Object[] record = records.get(1);
+            assertEquals("2014-10-02 22:15:01 UTC", record[0].toString());
+            assertEquals("140.36.216.47", record[1]);
+            assertEquals("/category/music?from=10", record[2]);
+            assertEquals("GET", record[3]);
+            assertEquals("-", record[4]);
+            assertEquals(200L, record[5]);
+            assertEquals("Mozilla/5.0", record[6]);
+            assertEquals("-", record[7]);
+            assertEquals(70L, record[8]);
         }
-        return reports.build();
     }
 
-    private static List<String> fileListToList(FileList list)
+    static List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
     {
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
-        for (int i=0; i < list.getTaskCount(); i++) {
-            for (String path : list.get(i)) {
-                builder.add(path);
-            }
-        }
-        return builder.build();
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        return Pages.toObjects(schema, output.pages);
     }
 }

data/src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java

@@ -2,6 +2,9 @@ package org.embulk.input.s3;
 
 import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.GetObjectRequest;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3Object;
+import com.amazonaws.services.s3.model.S3ObjectInputStream;
 import org.embulk.EmbulkTestRuntime;
 import org.embulk.input.s3.AbstractS3FileInputPlugin.S3InputStreamReopener;
 import org.junit.Before;
@@ -9,11 +12,11 @@ import org.junit.Rule;
 import org.junit.Test;
 
 import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
 import java.io.InputStream;
 import java.io.InputStreamReader;
 
 import static org.junit.Assert.assertEquals;
-import static org.embulk.input.s3.TestS3FileInputPlugin.s3object;
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.doThrow;
@@ -60,4 +63,14 @@ public class TestS3InputStreamReopener
             }
         }
     }
+
+    static S3Object s3object(String key, String value)
+    {
+        S3Object o = new S3Object();
+        o.setObjectContent(new S3ObjectInputStream(new ByteArrayInputStream(value.getBytes()), null));
+        ObjectMetadata om = new ObjectMetadata();
+        om.setContentLength(value.length());
+        o.setObjectMetadata(om);
+        return o;
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,3 @@
+timestamp,host,path,method,referer,code,agent,user,size
+2014-10-02 22:15:39,84.186.29.187,/category/electronics,GET,/category/music,200,Mozilla/5.0,-,136
+2014-10-02 22:15:01,140.36.216.47,/category/music?from=10,GET,-,200,Mozilla/5.0,-,70

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-s3
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-11-18 00:00:00.000000000 Z
+date: 2015-12-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,13 +52,15 @@ files:
 - src/main/java/org/embulk/input/s3/AwsCredentialsTask.java
 - src/main/java/org/embulk/input/s3/FileList.java
 - src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
+- src/test/java/org/embulk/input/s3/TestAwsCredentials.java
 - src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
 - src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
+- src/test/resources/sample_01.csv
 - classpath/aws-java-sdk-core-1.10.33.jar
 - classpath/aws-java-sdk-kms-1.10.33.jar
 - classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/commons-codec-1.6.jar
-- classpath/embulk-input-s3-0.2.4.jar
+- classpath/embulk-input-s3-0.2.5.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/jcl-over-slf4j-1.7.12.jar