embulk-input-gcs 0.3.0 → 0.3.1

data/src/main/java/org/embulk/input/gcs/SingleFileProvider.java

@@ -1,23 +1,19 @@
 package org.embulk.input.gcs;
 
-import com.google.api.client.util.IOUtils;
-import com.google.api.services.storage.Storage;
-import com.google.common.base.Throwables;
+import com.google.cloud.ReadChannel;
+import com.google.cloud.storage.Storage;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
-import org.embulk.spi.util.RetryExecutor;
+import org.embulk.spi.util.ResumableInputStream;
 import org.slf4j.Logger;
 
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
 import java.util.Iterator;
 
-import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+import static java.lang.String.format;
 
 public class SingleFileProvider
         implements InputStreamFileInput.Provider
@@ -25,20 +21,17 @@ public class SingleFileProvider
     private final Storage client;
     private final String bucket;
     private final Iterator<String> iterator;
-    private final int maxConnectionRetry;
     private boolean opened = false;
-    private final Logger log = Exec.getLogger(SingleFileProvider.class);
 
-    public SingleFileProvider(PluginTask task, int taskIndex)
+    SingleFileProvider(PluginTask task, int taskIndex)
     {
-        this.client = GcsFileInput.newGcsClient(task, GcsFileInput.newGcsAuth(task));
+        this.client = AuthUtils.newClient(task);
         this.bucket = task.getBucket();
         this.iterator = task.getFiles().get(taskIndex).iterator();
-        this.maxConnectionRetry = task.getMaxConnectionRetry();
     }
 
     @Override
-    public InputStreamWithHints openNextWithHints() throws IOException
+    public InputStreamWithHints openNextWithHints()
     {
         if (opened) {
             return null;
@@ -48,10 +41,9 @@ public class SingleFileProvider
             return null;
         }
         String key = iterator.next();
-        File tempFile = Exec.getTempFileSpace().createTempFile();
-        getRemoteContentsWithRetry(tempFile, client, bucket, key, maxConnectionRetry);
+        ReadChannel ch = client.get(bucket, key).reader();
         return new InputStreamWithHints(
-                new BufferedInputStream(new FileInputStream(tempFile)),
+                new ResumableInputStream(Channels.newInputStream(ch), new InputStreamReopener(client, bucket, key)),
                 String.format("gcs://%s/%s", bucket, key)
         );
     }
@@ -61,53 +53,28 @@ public class SingleFileProvider
     {
     }
 
-    private Void getRemoteContentsWithRetry(final File tempFile, final Storage client, final String bucket, final String key, int maxConnectionRetry)
+    static class InputStreamReopener
+            implements ResumableInputStream.Reopener
     {
-        try {
-            return retryExecutor()
-                    .withRetryLimit(maxConnectionRetry)
-                    .withInitialRetryWait(500)
-                    .withMaxRetryWait(30 * 1000)
-                    .runInterruptible(new RetryExecutor.Retryable<Void>() {
-                        @Override
-                        public Void call() throws IOException
-                        {
-                            Storage.Objects.Get getObject = client.objects().get(bucket, key);
-                            try (BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(tempFile))) {
-                                IOUtils.copy(getObject.executeMediaAsInputStream(), outputStream);
-                            }
-                            return null;
-                        }
+        private Logger logger = Exec.getLogger(getClass());
+        private final Storage client;
+        private final String bucket;
+        private final String key;
 
-                        @Override
-                        public boolean isRetryableException(Exception exception)
-                        {
-                            return true;  // TODO
-                        }
-
-                        @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-                                throws RetryExecutor.RetryGiveupException
-                        {
-                            String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
-                            if (retryCount % 3 == 0) {
-                                log.warn(message, exception);
-                            }
-                            else {
-                                log.warn(message);
-                            }
-                        }
-
-                        @Override
-                        public void onGiveup(Exception firstException, Exception lastException)
-                                throws RetryExecutor.RetryGiveupException
-                        {
-                        }
-                    });
+        InputStreamReopener(Storage client, String bucket, String key)
+        {
+            this.client = client;
+            this.bucket = bucket;
+            this.key = key;
         }
-        catch (RetryExecutor.RetryGiveupException | InterruptedException ex) {
-            throw Throwables.propagate(ex.getCause());
+
+        @Override
+        public InputStream reopen(long offset, Exception closedCause) throws IOException
+        {
+            logger.warn(format("GCS read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
+            ReadChannel ch = client.get(bucket, key).reader();
+            ch.seek(offset);
+            return Channels.newInputStream(ch);
         }
     }
 }

data/src/test/java/org/embulk/input/gcs/TestAuthUtils.java

@@ -0,0 +1,153 @@
+package org.embulk.input.gcs;
+
+import com.google.auth.Credentials;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.unit.LocalFile;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Mockito;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.security.GeneralSecurityException;
+import java.util.Optional;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assume.assumeNotNull;
+
+public class TestAuthUtils
+{
+    private static Optional<String> GCP_EMAIL;
+    private static Optional<String> GCP_P12_KEYFILE;
+    private static Optional<String> GCP_JSON_KEYFILE;
+    private static String GCP_BUCKET;
+    private static final String GCP_APPLICATION_NAME = "embulk-input-gcs";
+
+    /*
+     * This test case requires environment variables
+     *   GCP_EMAIL
+     *   GCP_P12_KEYFILE
+     *   GCP_JSON_KEYFILE
+     *   GCP_BUCKET
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        String gcpEmail = System.getenv("GCP_EMAIL");
+        String gcpP12KeyFile = System.getenv("GCP_P12_KEYFILE");
+        String gcpJsonKeyFile = System.getenv("GCP_JSON_KEYFILE");
+        String gcpBucket = System.getenv("GCP_BUCKET");
+
+        // skip test cases, if environment variables are not set.
+        assumeNotNull(gcpEmail, gcpP12KeyFile, gcpJsonKeyFile, gcpBucket);
+
+        GCP_EMAIL = Optional.of(gcpEmail);
+        GCP_P12_KEYFILE = Optional.of(gcpP12KeyFile);
+        GCP_JSON_KEYFILE = Optional.of(gcpJsonKeyFile);
+        GCP_BUCKET = gcpBucket;
+    }
+
+    @SuppressFBWarnings("URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD")
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private ConfigSource config;
+
+    @Before
+    public void setUp()
+    {
+        config = config();
+    }
+
+    @Test
+    public void testGetServiceAccountCredentialSuccess() throws IOException, GeneralSecurityException
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertThat(AuthUtils.fromP12(task), instanceOf(Credentials.class));
+    }
+
+    @Test(expected = FileNotFoundException.class)
+    public void testGetServiceAccountCredentialThrowFileNotFoundException()
+            throws IOException, GeneralSecurityException
+    {
+        Path mockNotFound = Mockito.mock(Path.class);
+        Mockito.when(mockNotFound.toString()).thenReturn("/path/to/notfound.p12");
+        LocalFile p12File = Mockito.mock(LocalFile.class);
+        Mockito.doReturn(mockNotFound).when(p12File).getPath();
+
+        PluginTask task = Mockito.mock(PluginTask.class);
+        Mockito.doReturn(Optional.of(p12File)).when(task).getP12Keyfile();
+        AuthUtils.fromP12(task);
+    }
+
+    @Test
+    public void testGetGcsClientUsingServiceAccountCredentialSuccess()
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertThat(AuthUtils.newClient(task), instanceOf(com.google.cloud.storage.Storage.class));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
+    {
+        PluginTask task = config.set("bucket", "non-exists-bucket")
+                .loadConfig(PluginTask.class);
+        AuthUtils.newClient(task);
+    }
+
+    @Test
+    public void testGetServiceAccountCredentialFromJsonFileSuccess()
+            throws IOException
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+        assertThat(AuthUtils.fromJson(task), instanceOf(Credentials.class));
+    }
+
+    @Test(expected = FileNotFoundException.class)
+    public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
+            throws IOException
+    {
+        Path mockNotFound = Mockito.mock(Path.class);
+        Mockito.when(mockNotFound.toString()).thenReturn("/path/to/notfound.json");
+        LocalFile jsonFile = Mockito.mock(LocalFile.class);
+        Mockito.doReturn(mockNotFound).when(jsonFile).getPath();
+
+        PluginTask task = Mockito.mock(PluginTask.class);
+        Mockito.doReturn(Optional.of(jsonFile)).when(task).getJsonKeyfile();
+        AuthUtils.fromJson(task);
+    }
+
+    @Test
+    public void testGetServiceAccountCredentialFromJsonSuccess()
+    {
+        PluginTask task = config.set("auth_method", AuthUtils.AuthMethod.json_key).loadConfig(PluginTask.class);
+        assertThat(AuthUtils.newClient(task), instanceOf(com.google.cloud.storage.Storage.class));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
+    {
+        PluginTask task = config.set("auth_method", AuthUtils.AuthMethod.json_key)
+                .set("bucket", "non-exists-bucket")
+                .loadConfig(PluginTask.class);
+        AuthUtils.newClient(task);
+    }
+
+    private ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("application_name", GCP_APPLICATION_NAME);
+    }
+}

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.input.gcs;
 
-import com.google.api.services.storage.Storage;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.Lists;
@@ -11,7 +10,6 @@ import org.embulk.config.ConfigSource;
 import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.FileInputRunner;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.Schema;
@@ -23,20 +21,15 @@ import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 
-import java.io.IOException;
-import java.security.GeneralSecurityException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Optional;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 import static org.junit.Assume.assumeNotNull;
 
-import java.lang.reflect.InvocationTargetException;
-
-import java.lang.reflect.Method;
-
 public class TestGcsFileInputPlugin
 {
     private static Optional<String> GCP_EMAIL;
@@ -45,8 +38,7 @@ public class TestGcsFileInputPlugin
     private static String GCP_BUCKET;
     private static String GCP_BUCKET_DIRECTORY;
     private static String GCP_PATH_PREFIX;
-    private static String GCP_APPLICATION_NAME;
-    private static int MAX_CONNECTION_RETRY = 3;
+    private static String GCP_APPLICATION_NAME = "embulk-input-gcs";
     private FileInputRunner runner;
     private MockPageOutput output;
 
@@ -83,7 +75,7 @@ public class TestGcsFileInputPlugin
     private GcsFileInputPlugin plugin;
 
     @Before
-    public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
+    public void createResources()
     {
         config = config();
         plugin = new GcsFileInputPlugin();
@@ -99,7 +91,7 @@ public class TestGcsFileInputPlugin
                 .set("path_prefix", "my-prefix");
 
         PluginTask task = config.loadConfig(PluginTask.class);
-        assertEquals(true, task.getIncremental());
+        assertTrue(task.getIncremental());
         assertEquals("private_key", task.getAuthMethod().toString());
         assertEquals("Embulk GCS input plugin", task.getApplicationName());
     }
@@ -215,22 +207,15 @@ public class TestGcsFileInputPlugin
 
     @Test
     public void testGcsClientCreateSuccessfully()
-            throws GeneralSecurityException, IOException, NoSuchMethodException,
-            IllegalAccessException, InvocationTargetException
     {
         PluginTask task = config().loadConfig(PluginTask.class);
-        runner.transaction(config, new Control());
-
-        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task)); // no errors happens
+        AuthUtils.newClient(task);
     }
 
     @Test(expected = ConfigException.class)
     public void testGcsClientCreateThrowConfigException()
-            throws GeneralSecurityException, IOException, NoSuchMethodException,
-            IllegalAccessException, InvocationTargetException
     {
+        // verify AuthUtils#newClient() to throws ConfigException for non-exists-bucket
         ConfigSource config = Exec.newConfigSource()
                 .set("bucket", "non-exists-bucket")
                 .set("path_prefix", "my-prefix")
@@ -240,11 +225,7 @@ public class TestGcsFileInputPlugin
                 .set("parser", parserConfig(schemaConfig()));
 
         PluginTask task = config.loadConfig(PluginTask.class);
-        runner.transaction(config, new Control());
-
-        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
+        AuthUtils.newClient(task);
     }
 
     @Test
@@ -254,14 +235,7 @@ public class TestGcsFileInputPlugin
         FileList.Builder builder = new FileList.Builder(config);
         builder.add("in/aa/a", 1);
         task.setFiles(builder.build());
-        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
-        {
-            @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
-            {
-                return emptyTaskReports(taskCount);
-            }
-        });
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, (taskSource, taskCount) -> emptyTaskReports(taskCount));
         assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
     }
 
@@ -269,12 +243,11 @@ public class TestGcsFileInputPlugin
     public void testCleanup()
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+        plugin.cleanup(task.dump(), 0, Lists.newArrayList()); // no errors happens
     }
 
     @Test
     public void testListFilesByPrefix()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
         List<String> expected = Arrays.asList(
                 GCP_BUCKET_DIRECTORY + "sample_01.csv",
@@ -282,21 +255,12 @@ public class TestGcsFileInputPlugin
         );
 
         PluginTask task = config.loadConfig(PluginTask.class);
-        ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
-            @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
-            {
-                assertEquals(2, taskCount);
-                return emptyTaskReports(taskCount);
-            }
+        ConfigDiff configDiff = plugin.transaction(config, (taskSource, taskCount) -> {
+            assertEquals(2, taskCount);
+            return emptyTaskReports(taskCount);
         });
 
-        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
-        FileList.Builder builder = new FileList.Builder(config);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.empty());
-        FileList fileList = builder.build();
+        FileList fileList = GcsFileInput.listFiles(task);
         assertEquals(expected.get(0), fileList.get(0).get(0));
         assertEquals(expected.get(1), fileList.get(1).get(0));
         assertEquals(GCP_BUCKET_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
@@ -304,7 +268,6 @@ public class TestGcsFileInputPlugin
 
     @Test
     public void testListFilesByPrefixWithPattern()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
         List<String> expected = Arrays.asList(
                 GCP_BUCKET_DIRECTORY + "sample_01.csv"
@@ -312,27 +275,18 @@ public class TestGcsFileInputPlugin
 
         ConfigSource configWithPattern = config.deepCopy().set("path_match_pattern", "1");
         PluginTask task = configWithPattern.loadConfig(PluginTask.class);
-        ConfigDiff configDiff = plugin.transaction(configWithPattern, new FileInputPlugin.Control() {
-            @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
-            {
-                assertEquals(1, taskCount);
-                return emptyTaskReports(taskCount);
-            }
+        ConfigDiff configDiff = plugin.transaction(configWithPattern, (taskSource, taskCount) -> {
+            assertEquals(1, taskCount);
+            return emptyTaskReports(taskCount);
         });
 
-        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
-        FileList.Builder builder = new FileList.Builder(configWithPattern);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.empty());
-        FileList fileList = builder.build();
+        FileList fileList = GcsFileInput.listFiles(task);
         assertEquals(expected.get(0), fileList.get(0).get(0));
         assertEquals(GCP_BUCKET_DIRECTORY + "sample_01.csv", configDiff.get(String.class, "last_path"));
     }
 
     @Test
-    public void testListFilesByPrefixIncrementalFalse() throws Exception
+    public void testListFilesByPrefixIncrementalFalse()
     {
         ConfigSource config = config().deepCopy()
                 .set("incremental", false);
@@ -342,22 +296,22 @@ public class TestGcsFileInputPlugin
         assertEquals("{}", configDiff.toString());
     }
 
-    @Test
+    @Test(expected = ConfigException.class)
     public void testListFilesByPrefixNonExistsBucket()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
-        PluginTask task = config.loadConfig(PluginTask.class);
+        PluginTask task = config
+                .set("bucket", "non-exists-bucket")
+                .set("path_prefix", "prefix")
+                .loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
 
-        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
-        FileList.Builder builder = new FileList.Builder(config);
-        GcsFileInput.listGcsFilesByPrefix(builder, client, "non-exists-bucket", "prefix", Optional.empty()); // no errors happens
+        // after refactoring, GcsFileInput#listFiles() won't accept initialized client
+        // hence, this test will throw ConfigException
+        GcsFileInput.listFiles(task);
     }
 
     @Test
-    public void testNonExistingFilesWithPathPrefix() throws Exception
+    public void testNonExistingFilesWithPathPrefix()
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("bucket", GCP_BUCKET)
@@ -394,7 +348,6 @@ public class TestGcsFileInputPlugin
 
     @Test
     public void testGcsFileInputByOpen()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("bucket", GCP_BUCKET)
@@ -407,31 +360,24 @@ public class TestGcsFileInputPlugin
         PluginTask task = config.loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
 
-        Method method = GcsFileInput.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
-        method.setAccessible(true);
-        Storage client = GcsFileInput.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
-        task.setFiles(GcsFileInput.listFiles(task, client));
+        task.setFiles(GcsFileInput.listFiles(task));
 
         assertRecords(config, output);
     }
 
     @Test
     public void testBase64()
-            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
-        Method method = GcsFileInput.class.getDeclaredMethod("base64Encode", String.class);
-        method.setAccessible(true);
-
-        assertEquals("CgFj", method.invoke(plugin, "c"));
-        assertEquals("CgJjMg==", method.invoke(plugin, "c2"));
-        assertEquals("Cgh0ZXN0LmNzdg==", method.invoke(plugin, "test.csv"));
-        assertEquals("ChZnY3MtdGVzdC9zYW1wbGVfMDEuY3N2", method.invoke(plugin, "gcs-test/sample_01.csv"));
+        assertEquals("CgFj", GcsFileInput.base64Encode("c"));
+        assertEquals("CgJjMg==", GcsFileInput.base64Encode("c2"));
+        assertEquals("Cgh0ZXN0LmNzdg==", GcsFileInput.base64Encode("test.csv"));
+        assertEquals("ChZnY3MtdGVzdC9zYW1wbGVfMDEuY3N2", GcsFileInput.base64Encode("gcs-test/sample_01.csv"));
         String params = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc127";
         String expected = "Cn9jY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjMTI3";
-        assertEquals(expected, method.invoke(plugin, params));
+        assertEquals(expected, GcsFileInput.base64Encode(params));
     }
 
-    public ConfigSource config()
+    private ConfigSource config()
     {
         return Exec.newConfigSource()
                 .set("bucket", GCP_BUCKET)
@@ -444,7 +390,7 @@ public class TestGcsFileInputPlugin
                 .set("parser", parserConfig(schemaConfig()));
     }
 
-    static List<TaskReport> emptyTaskReports(int taskCount)
+    private static List<TaskReport> emptyTaskReports(int taskCount)
     {
         ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
         for (int i = 0; i < taskCount; i++) {
@@ -525,11 +471,13 @@ public class TestGcsFileInputPlugin
 
     private static String getDirectory(String dir)
     {
-        if (dir != null && !dir.endsWith("/")) {
-            dir = dir + "/";
-        }
-        if (dir.startsWith("/")) {
-            dir = dir.replaceFirst("/", "");
+        if (dir != null) {
+            if (!dir.endsWith("/")) {
+                dir = dir + "/";
+            }
+            if (dir.startsWith("/")) {
+                dir = dir.replaceFirst("/", "");
+            }
         }
         return dir;
     }