embulk-input-gcs 0.1.7 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24b44f4db77ef29422e03052b58cdf67378d5a79
-  data.tar.gz: 031a0686510df86a6b87780aa0da707cd7cd64cc
+  metadata.gz: 219d292e87e6a2b326144302280e10bba7e3ea2a
+  data.tar.gz: c7e6e1933bc1961e16f18c68022261e1948cd5bc
 SHA512:
-  metadata.gz: eb5815752627aeef5c32de944d23e78c03313970e574f61e3706808bb4314643311e4d581ae2fc7589ede24e80ff53236cdd0defecbd647a2d382d26b3630dcc
-  data.tar.gz: 5ce214a4e252f78bbc9a9215d3edcc0de671220e2f5f9697687e4b4ac0d9d05303a5a724f399ad1a8175a8a8d85bd50d977fc01af8a718e74cb50981e34b3b6d
+  metadata.gz: e09e566db719125bc872a33026c4dd15bb1b29bf3ad82ce75a5cdd1fc5c342e2c8c72179accd3b9fafecf8ca3945dd26c0e1cdc70036d3a82d509485c388e925
+  data.tar.gz: aa068f8ac05d9fafec71baa24d71f18b59096521277ce836a5b27c1f1c491d4aee8ffe004d71117d5d6d68482e6525071fbcb2da4f3ca30601c638582cf7d260

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.1.8 - 2015-10-29
+
+* Added unit tests
+
 Release 0.1.7 - 2015-10-06
 
 * Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account

data/README.md

@@ -136,7 +136,7 @@ in:
 
 On the other hand, you don't need to explicitly create a service account for embulk when you
 run embulk in Google Compute Engine. In this third authentication method, you need to
-add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
+add the API scope "https://www.googleapis.com/auth/devstorage.read_only" to the scope list of your
 Compute Engine VM instance, then you can configure embulk like this.
 
 [Setting the scope of service account access for instances](https://cloud.google.com/compute/docs/authentication)
@@ -153,3 +153,52 @@ in:
 ./gradlew gem
 ```
 
+## Test
+
+To run unit tests, we need to configure the following environment variables.
+
+Additionally, following files will be needed to upload to existing GCS bucket.
+* [sample_01.csv](./src/test/resources/sample_01.csv)
+* [sample_02.csv](./src/test/resources/sample_02.csv)
+
+When environment variables are not set, skip some test cases.
+
+```
+GCP_EMAIL
+GCP_P12_KEYFILE
+GCP_JSON_KEYFILE
+GCP_BUCKET
+GCP_BUCKET_DIRECTORY(optional, if needed)
+```
+
+If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
+```
+$ vi ~/Library/LaunchAgents/environment.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>my.startup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>sh</string>
+    <string>-c</string>
+    <string>
+      launchctl setenv GCP_EMAIL ABCXYZ123ABCXYZ123.gserviceaccount.com
+      launchctl setenv GCP_P12_KEYFILE /path/to/p12_keyfile.p12
+      launchctl setenv GCP_JSON_KEYFILE /path/to/json_keyfile.json
+      launchctl setenv GCP_BUCKET my-bucket
+      launchctl setenv GCP_BUCKET_DIRECTORY unittests
+    </string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+</dict>
+</plist>
+
+$ launchctl load ~/Library/LaunchAgents/environment.plist
+$ launchctl getenv GCP_EMAIL //try to get value.
+
+Then start your applications.
+```

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -15,16 +16,18 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.7"
+version = "0.1.8"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.7.1"
-    provided "org.embulk:embulk-core:0.7.1"
+    compile  "org.embulk:embulk-core:0.7.5"
+    provided "org.embulk:embulk-core:0.7.5"
 
     compile "com.google.http-client:google-http-client-jackson2:1.19.0"
     compile ("com.google.apis:google-api-services-storage:v1-rev27-1.19.1") {exclude module: "guava-jdk5"}
 
-    testCompile "junit:junit:4.+"
+    testCompile "junit:junit:4.12"
+    testCompile "org.embulk:embulk-core:0.7.5:tests"
+    testCompile "org.embulk:embulk-standards:0.7.5"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -165,7 +165,7 @@ public class GcsFileInputPlugin
     {
     }
 
-    private static Storage newGcsClient(final PluginTask task)
+    protected Storage newGcsClient(final PluginTask task)
     {
         Storage client = null;
         try {
@@ -213,12 +213,11 @@ public class GcsFileInputPlugin
             Storage.Buckets.Get getBucket = client.buckets().get(bucket);
             getBucket.setProjection("full");
             Bucket bk = getBucket.execute();
-            if (log.isDebugEnabled()) {
-                log.debug("bucket name: " + bucket);
-                log.debug("bucket location: " + bk.getLocation());
-                log.debug("bucket timeCreated: " + bk.getTimeCreated());
-                log.debug("bucket owner: " + bk.getOwner());
-            }
+
+            log.debug("bucket name: " + bucket);
+            log.debug("bucket location: " + bk.getLocation());
+            log.debug("bucket timeCreated: " + bk.getTimeCreated());
+            log.debug("bucket owner: " + bk.getOwner());
         } catch (IOException e) {
             log.warn("Could not access to bucket:" + bucket);
             log.warn(e.getMessage());
@@ -238,13 +237,11 @@ public class GcsFileInputPlugin
                     break;
                 }
                 for (StorageObject o : items) {
-                    if (log.isDebugEnabled()) {
-                        log.debug("filename: " + o.getName());
-                        log.debug("updated: " + o.getUpdated());
-                    }
                     if (o.getSize().compareTo(BigInteger.ZERO) > 0) {
                         builder.add(o.getName());
                     }
+                    log.debug("filename: " + o.getName());
+                    log.debug("updated: " + o.getUpdated());
                 }
                 lastKey = objects.getNextPageToken();
                 listObjects.setPageToken(lastKey);
@@ -264,41 +261,10 @@ public class GcsFileInputPlugin
         return new GcsFileInput(task, taskIndex);
     }
 
-    public static class GcsFileInput
+    public class GcsFileInput
             extends InputStreamFileInput
             implements TransactionalFileInput
     {
-        private static class SingleFileProvider
-                implements InputStreamFileInput.Provider
-        {
-            private final Storage client;
-            private final String bucket;
-            private final String key;
-            private boolean opened = false;
-
-            public SingleFileProvider(PluginTask task, int taskIndex)
-            {
-                this.client = newGcsClient(task);
-                this.bucket = task.getBucket();
-                this.key = task.getFiles().get(taskIndex);
-            }
-
-            @Override
-            public InputStream openNext() throws IOException
-            {
-                if (opened) {
-                    return null;
-                }
-                opened = true;
-                Storage.Objects.Get getObject = client.objects().get(bucket, key);
-
-                return getObject.executeMediaAsInputStream();
-            }
-
-            @Override
-            public void close() { }
-        }
-
         public GcsFileInput(PluginTask task, int taskIndex)
         {
             super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
@@ -315,6 +281,37 @@ public class GcsFileInputPlugin
         public void close() { }
     }
 
+    private class SingleFileProvider
+            implements InputStreamFileInput.Provider
+    {
+        private final Storage client;
+        private final String bucket;
+        private final String key;
+        private boolean opened = false;
+
+        public SingleFileProvider(PluginTask task, int taskIndex)
+        {
+            this.client = newGcsClient(task);
+            this.bucket = task.getBucket();
+            this.key = task.getFiles().get(taskIndex);
+        }
+
+        @Override
+        public InputStream openNext() throws IOException
+        {
+            if (opened) {
+                return null;
+            }
+            opened = true;
+            Storage.Objects.Get getObject = client.objects().get(bucket, key);
+
+            return getObject.executeMediaAsInputStream();
+        }
+
+        @Override
+        public void close() { }
+    }
+
     public enum AuthMethod
     {
         private_key("private_key"),

data/src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java

@@ -0,0 +1,177 @@
+package org.embulk.input.gcs;
+
+import com.google.common.base.Optional;
+import org.embulk.EmbulkTestRuntime;
+import com.google.api.services.storage.Storage;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.security.GeneralSecurityException;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+
+import java.lang.reflect.Field;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+public class TestGcsAuthentication
+{
+    private static Optional<String> GCP_EMAIL;
+    private static Optional<String> GCP_P12_KEYFILE;
+    private static Optional<String> GCP_JSON_KEYFILE;
+    private static String GCP_BUCKET;
+    private static final String GCP_APPLICATION_NAME = "embulk-input-gcs";
+
+    /*
+     * This test case requires environment variables
+     *   GCP_EMAIL
+     *   GCP_P12_KEYFILE
+     *   GCP_JSON_KEYFILE
+     *   GCP_BUCKET
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
+        GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
+        GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+        GCP_BUCKET = System.getenv("GCP_BUCKET");
+        // skip test cases, if environment variables are not set.
+        assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+    @Test
+    public void testGetServiceAccountCredentialSuccess()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "private_key",
+                GCP_EMAIL,
+                GCP_P12_KEYFILE,
+                null,
+                GCP_APPLICATION_NAME
+        );
+
+        Field field = GcsAuthentication.class.getDeclaredField("credentials");
+        field.setAccessible(true);
+
+        assertEquals(GoogleCredential.class, field.get(auth).getClass());
+    }
+
+    @Test(expected = FileNotFoundException.class)
+    public void testGetServiceAccountCredentialThrowFileNotFoundException()
+            throws GeneralSecurityException, IOException
+    {
+        Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
+        GcsAuthentication auth = new GcsAuthentication(
+                "private_key",
+                GCP_EMAIL,
+                notFoundP12Keyfile,
+                null,
+                GCP_APPLICATION_NAME
+        );
+    }
+
+    @Test
+    public void testGetGcsClientUsingServiceAccountCredentialSuccess()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "private_key",
+                GCP_EMAIL,
+                GCP_P12_KEYFILE,
+                null,
+                GCP_APPLICATION_NAME
+        );
+
+        Storage client = auth.getGcsClient(GCP_BUCKET);
+
+        assertEquals(Storage.class, client.getClass());
+    }
+
+    @Test(expected = GoogleJsonResponseException.class)
+    public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "private_key",
+                GCP_EMAIL,
+                GCP_P12_KEYFILE,
+                null,
+                GCP_APPLICATION_NAME
+        );
+
+        Storage client = auth.getGcsClient("non-exists-bucket");
+
+        assertEquals(Storage.class, client.getClass());
+    }
+
+    @Test
+    public void testGetServiceAccountCredentialFromJsonFileSuccess()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "json_key",
+                GCP_EMAIL,
+                null,
+                GCP_JSON_KEYFILE,
+                GCP_APPLICATION_NAME
+        );
+        Field field = GcsAuthentication.class.getDeclaredField("credentials");
+        field.setAccessible(true);
+
+        assertEquals(GoogleCredential.class, field.get(auth).getClass());
+    }
+
+    @Test(expected = FileNotFoundException.class)
+    public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
+            throws GeneralSecurityException, IOException
+    {
+        Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
+        GcsAuthentication auth = new GcsAuthentication(
+                "json_key",
+                GCP_EMAIL,
+                null,
+                notFoundJsonKeyfile,
+                GCP_APPLICATION_NAME
+        );
+    }
+
+    @Test
+    public void testGetServiceAccountCredentialFromJsonSuccess()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "json_key",
+                GCP_EMAIL,
+                null,
+                GCP_JSON_KEYFILE,
+                GCP_APPLICATION_NAME
+        );
+
+        Storage client = auth.getGcsClient(GCP_BUCKET);
+
+        assertEquals(Storage.class, client.getClass());
+    }
+
+    @Test(expected = GoogleJsonResponseException.class)
+    public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
+            throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+    {
+        GcsAuthentication auth = new GcsAuthentication(
+                "json_key",
+                GCP_EMAIL,
+                null,
+                GCP_JSON_KEYFILE,
+                GCP_APPLICATION_NAME
+        );
+
+        Storage client = auth.getGcsClient("non-exists-bucket");
+    }
+}

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -1,9 +1,355 @@
 package org.embulk.input.gcs;
 
-import static org.junit.Assert.*;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.util.Pages;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.input.gcs.GcsFileInputPlugin.PluginTask;
+import org.embulk.standards.CsvParserPlugin;
+
+import com.google.api.services.storage.Storage;
+
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+public class TestGcsFileInputPlugin
+{
+    private static Optional<String> GCP_EMAIL;
+    private static Optional<String> GCP_P12_KEYFILE;
+    private static Optional<String> GCP_JSON_KEYFILE;
+    private static String GCP_BUCKET;
+    private static String GCP_BUCKET_DIRECTORY;
+    private static String GCP_PATH_PREFIX;
+    private final String GCP_APPLICATION_NAME = "embulk-input-gcs";
+    private FileInputRunner runner;
+    private MockPageOutput output;
+
+    /*
+     * This test case requires environment variables
+     *   GCP_EMAIL
+     *   GCP_P12_KEYFILE
+     *   GCP_JSON_KEYFILE
+     *   GCP_BUCKET
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
+        GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
+        GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+        GCP_BUCKET = System.getenv("GCP_BUCKET");
+        // skip test cases, if environment variables are not set.
+        assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
+
+        GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
+        GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private ConfigSource config;
+    private GcsFileInputPlugin plugin;
+
+    @Before
+    public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
+    {
+        config = config();
+        plugin = new GcsFileInputPlugin();
+        runner = new FileInputRunner(runtime.getInstance(GcsFileInputPlugin.class));
+        output = new MockPageOutput();
+    }
+
+    @Test
+    public void checkDefaultValues()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix");
+
+        GcsFileInputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals("private_key", task.getAuthMethod().toString());
+        assertEquals("Embulk GCS input plugin", task.getApplicationName());
+    }
+
+    // p12_keyfile is null when auth_method is private_key
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesP12keyNull()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", null);
+
+        runner.transaction(config, new Control());
+    }
+
+    // both p12_keyfile and p12_keyfile_fullpath set
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesConflictSetting()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("p12_keyfile_fullpath", GCP_P12_KEYFILE);
+
+        runner.transaction(config, new Control());
+    }
+
+    // invalid p12keyfile when auth_method is private_key
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesInvalidPrivateKey()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", "invalid-key.p12");
+
+        runner.transaction(config, new Control());
+    }
+
+    // json_keyfile is null when auth_method is json_key
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesJsonKeyfileNull()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "json_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("json_keyfile", null);
+
+        runner.transaction(config, new Control());
+    }
+
+    @Test
+    public void testGcsClientCreateSuccessfully()
+            throws GeneralSecurityException, IOException, NoSuchFieldException
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+        plugin.newGcsClient(task); // no errors happens
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGcsClientCreateThrowConfigException()
+            throws GeneralSecurityException, IOException, NoSuchFieldException
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", "non-exists-bucket")
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "json_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("json_keyfile", GCP_JSON_KEYFILE);
+
+        PluginTask task = config().loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+        plugin.newGcsClient(task);
+    }
+
+    @Test
+    public void testResume()
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        task.setFiles(Arrays.asList(new String[]{"in/aa/a"}));
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                return emptyTaskReports(taskCount);
+            }
+        });
+        assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+    }
+
+    @Test
+    public void testListFilesByPrefix()
+    {
+        List<String> expected = Arrays.asList(
+                GCP_BUCKET_DIRECTORY + "sample_01.csv",
+                GCP_BUCKET_DIRECTORY + "sample_02.csv"
+        );
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount) {
+                assertEquals(2, taskCount);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        Storage client = plugin.newGcsClient(task);
+        List<String> actual = plugin.listGcsFilesByPrefix(client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.<String>absent());
+        assertEquals(expected, actual);
+
+        assertEquals(GCP_BUCKET_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
+    }
+
+    @Test
+    public void testListFilesByPrefixNonExistsBucket()
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+        Storage client = plugin.newGcsClient(task);
+        plugin.listGcsFilesByPrefix(client, "non-exists-bucket", "prefix", Optional.<String>absent()); // no errors happens
+    }
+
+    @Test
+    public void testGcsFileInputByOpen() throws IOException
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", GCP_PATH_PREFIX)
+                .set("auth_method", "json_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
+
+        PluginTask task = config.loadConfig(PluginTask.class);
+        runner.transaction(config, new Control());
+        task.setFiles(plugin.listFiles(task));
+
+        assertRecords(config, output);
+    }
+
+    public ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", GCP_PATH_PREFIX)
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("application_name", GCP_APPLICATION_NAME)
+                .set("parser", parserConfig(schemaConfig()));
+    }
+
+    static List<TaskReport> emptyTaskReports(int taskCount)
+    {
+        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
+        for (int i = 0; i < taskCount; i++) {
+            reports.add(Exec.newTaskReport());
+        }
+        return reports.build();
+    }
+
+    private class Control
+            implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(runner.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "long"));
+        builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        return builder.build();
+    }
+
+    private void assertRecords(ConfigSource config, MockPageOutput output)
+    {
+        List<Object[]> records = getRecords(config, output);
+        assertEquals(8, records.size());
+        {
+            Object[] record = records.get(0);
+            assertEquals(1L, record[0]);
+            assertEquals(32864L, record[1]);
+            assertEquals("2015-01-27 19:23:49 UTC", record[2].toString());
+            assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
+            assertEquals("embulk", record[4]);
+        }
+
+        {
+            Object[] record = records.get(1);
+            assertEquals(2L, record[0]);
+            assertEquals(14824L, record[1]);
+            assertEquals("2015-01-27 19:01:23 UTC", record[2].toString());
+            assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
+            assertEquals("embulk jruby", record[4]);
+        }
+    }
+
+    private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
+    {
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        return Pages.toObjects(schema, output.pages);
+    }
 
-// @todo Write unit test.
-public class TestGcsFileInputPlugin {
+    private static String getDirectory(String dir)
+    {
+        if (dir != null && !dir.endsWith("/")) {
+            dir = dir + "/";
+        }
+        if (dir.startsWith("/")) {
+            dir = dir.replaceFirst("/", "");
+        }
+        return dir;
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/src/test/resources/sample_02.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.1.8
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-06 00:00:00.000000000 Z
+date: 2015-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -57,10 +57,13 @@ files:
 - settings.gradle
 - src/main/java/org/embulk/input/gcs/GcsAuthentication.java
 - src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java
+- src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java
 - src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.7.jar
+- classpath/embulk-input-gcs-0.1.8.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar