embulk-input-gcs 0.1.8 → 0.1.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 219d292e87e6a2b326144302280e10bba7e3ea2a
-  data.tar.gz: c7e6e1933bc1961e16f18c68022261e1948cd5bc
+  metadata.gz: d25891998dc1d0794b0fcfac04ea7a011f777f9c
+  data.tar.gz: cd7d9977e6bfa8df5212181ba49aaacf7998110b
 SHA512:
-  metadata.gz: e09e566db719125bc872a33026c4dd15bb1b29bf3ad82ce75a5cdd1fc5c342e2c8c72179accd3b9fafecf8ca3945dd26c0e1cdc70036d3a82d509485c388e925
-  data.tar.gz: aa068f8ac05d9fafec71baa24d71f18b59096521277ce836a5b27c1f1c491d4aee8ffe004d71117d5d6d68482e6525071fbcb2da4f3ca30601c638582cf7d260
+  metadata.gz: 9beb8bfab93341c05b076fb1937096cb86ab850da685806cd1aa29cf65ffea356e318bfdddc566321499e4ffa8b5ccac9c8028b0dd83aa20f29a6955c8556e9a
+  data.tar.gz: d3f872c0dd1460e5234b836761fbd7ea5f723b17371d881e4a91989b62edd10cfa98a0110eb26af5940c684a318164b1448f0ccd73962bc5db126b5c60070e8b

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.1.9 - 2015-10-30
+
+Fix GcsAuthentication object initialization for mapreduce executor. @muga thanks!
+
 Release 0.1.8 - 2015-10-29
 
 * Added unit tests

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.8"
+version = "0.1.9"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -86,7 +86,6 @@ public class GcsFileInputPlugin
     }
 
     private static final Logger log = Exec.getLogger(GcsFileInputPlugin.class);
-    private static GcsAuthentication auth;
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
@@ -115,8 +114,18 @@ public class GcsFileInputPlugin
             }
         }
 
+        Storage client = newGcsClient(task, newGcsAuth(task));
+
+        // list files recursively
+        task.setFiles(listFiles(task, client));
+        // number of processors is same with number of files
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    private GcsAuthentication newGcsAuth(PluginTask task)
+    {
         try {
-            auth = new GcsAuthentication(
+            return new GcsAuthentication(
                     task.getAuthMethod().getString(),
                     task.getServiceAccountEmail(),
                     task.getP12Keyfile().transform(localFileToPathString()),
@@ -126,11 +135,6 @@ public class GcsFileInputPlugin
         } catch (GeneralSecurityException | IOException ex) {
             throw new ConfigException(ex);
         }
-
-        // list files recursively
-        task.setFiles(listFiles(task));
-        // number of processors is same with number of files
-        return resume(task.dump(), task.getFiles().size(), control);
     }
 
     @Override
@@ -165,7 +169,7 @@ public class GcsFileInputPlugin
     {
     }
 
-    protected Storage newGcsClient(final PluginTask task)
+    protected Storage newGcsClient(final PluginTask task, final GcsAuthentication auth)
     {
         Storage client = null;
         try {
@@ -188,9 +192,8 @@ public class GcsFileInputPlugin
         };
     }
 
-    public List<String> listFiles(PluginTask task)
+    public List<String> listFiles(PluginTask task, Storage client)
     {
-        Storage client = newGcsClient(task);
         String bucket = task.getBucket();
 
         return listGcsFilesByPrefix(client, bucket, task.getPathPrefix(), task.getLastPath());
@@ -291,7 +294,7 @@ public class GcsFileInputPlugin
 
         public SingleFileProvider(PluginTask task, int taskIndex)
         {
-            this.client = newGcsClient(task);
+            this.client = newGcsClient(task, newGcsAuth(task));
             this.bucket = task.getBucket();
             this.key = task.getFiles().get(taskIndex);
         }

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -34,6 +34,8 @@ import org.junit.Rule;
 import org.junit.Test;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assume.assumeNotNull;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
 
 public class TestGcsFileInputPlugin
 {
@@ -103,7 +105,8 @@ public class TestGcsFileInputPlugin
                 .set("path_prefix", "my-prefix")
                 .set("auth_method", "private_key")
                 .set("service_account_email", GCP_EMAIL)
-                .set("p12_keyfile", null);
+                .set("p12_keyfile", null)
+                .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
     }
@@ -118,7 +121,8 @@ public class TestGcsFileInputPlugin
                 .set("auth_method", "private_key")
                 .set("service_account_email", GCP_EMAIL)
                 .set("p12_keyfile", GCP_P12_KEYFILE)
-                .set("p12_keyfile_fullpath", GCP_P12_KEYFILE);
+                .set("p12_keyfile_fullpath", GCP_P12_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
     }
@@ -132,7 +136,8 @@ public class TestGcsFileInputPlugin
                 .set("path_prefix", "my-prefix")
                 .set("auth_method", "private_key")
                 .set("service_account_email", GCP_EMAIL)
-                .set("p12_keyfile", "invalid-key.p12");
+                .set("p12_keyfile", "invalid-key.p12")
+                .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
     }
@@ -146,34 +151,44 @@ public class TestGcsFileInputPlugin
                 .set("path_prefix", "my-prefix")
                 .set("auth_method", "json_key")
                 .set("service_account_email", GCP_EMAIL)
-                .set("json_keyfile", null);
+                .set("json_keyfile", null)
+                .set("parser", parserConfig(schemaConfig()));
 
         runner.transaction(config, new Control());
     }
 
     @Test
     public void testGcsClientCreateSuccessfully()
-            throws GeneralSecurityException, IOException, NoSuchFieldException
+            throws GeneralSecurityException, IOException, NoSuchMethodException,
+            IllegalAccessException, InvocationTargetException
     {
         PluginTask task = config().loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
-        plugin.newGcsClient(task); // no errors happens
+
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task)); // no errors happens
     }
 
     @Test(expected = ConfigException.class)
     public void testGcsClientCreateThrowConfigException()
-            throws GeneralSecurityException, IOException, NoSuchFieldException
+            throws GeneralSecurityException, IOException, NoSuchMethodException,
+            IllegalAccessException, InvocationTargetException
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("bucket", "non-exists-bucket")
                 .set("path_prefix", "my-prefix")
                 .set("auth_method", "json_key")
                 .set("service_account_email", GCP_EMAIL)
-                .set("json_keyfile", GCP_JSON_KEYFILE);
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
 
         PluginTask task = config().loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
-        plugin.newGcsClient(task);
+
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
     }
 
     @Test
@@ -201,6 +216,7 @@ public class TestGcsFileInputPlugin
 
     @Test
     public void testListFilesByPrefix()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
         List<String> expected = Arrays.asList(
                 GCP_BUCKET_DIRECTORY + "sample_01.csv",
@@ -216,24 +232,30 @@ public class TestGcsFileInputPlugin
             }
         });
 
-        Storage client = plugin.newGcsClient(task);
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        Storage client = plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
         List<String> actual = plugin.listGcsFilesByPrefix(client, GCP_BUCKET, GCP_PATH_PREFIX, Optional.<String>absent());
         assertEquals(expected, actual);
-
         assertEquals(GCP_BUCKET_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
     }
 
     @Test
     public void testListFilesByPrefixNonExistsBucket()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
     {
         PluginTask task = config.loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
-        Storage client = plugin.newGcsClient(task);
+
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        Storage client = plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
         plugin.listGcsFilesByPrefix(client, "non-exists-bucket", "prefix", Optional.<String>absent()); // no errors happens
     }
 
     @Test
-    public void testGcsFileInputByOpen() throws IOException
+    public void testGcsFileInputByOpen()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("bucket", GCP_BUCKET)
@@ -245,7 +267,11 @@ public class TestGcsFileInputPlugin
 
         PluginTask task = config.loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
-        task.setFiles(plugin.listFiles(task));
+
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("newGcsAuth", PluginTask.class);
+        method.setAccessible(true);
+        Storage client = plugin.newGcsClient(task, (GcsAuthentication) method.invoke(plugin, task));
+        task.setFiles(plugin.listFiles(task, client));
 
         assertRecords(config, output);
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.1.9
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-28 00:00:00.000000000 Z
+date: 2015-10-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.8.jar
+- classpath/embulk-input-gcs-0.1.9.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar