embulk-input-gcs 0.1.13 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5e7dc0a163123cb7bb2de1738c46d60b70282d12
-  data.tar.gz: 10370c8b104b51f6e5ed55297181647a9caa1d82
+  metadata.gz: 84c5f3e02ea5ad9dd6b8536ad9c35bc5e893f785
+  data.tar.gz: 6e5cc8dda9c62e0515db17cf1c486b902609b853
 SHA512:
-  metadata.gz: 107ca80b29d9f3a0ea45a5d62326c570e8df3c2efea0a979f51cbcca174e8d67dd8d97183ad7856362699fbd693363ba96e10f9a0513122019a76d6e75dffeab
-  data.tar.gz: 39bb76c59dd27373c31280ad20b614d4fa6aab4655eacf9631b29a160838e545d0ec04d3ae7ea140c3897a6020c60f3e4623846bce43edf133708330b4ecd6b4
+  metadata.gz: ad92f9ff0c884436d4e667d01f23ea088e6a68793941ec8efb87e7694d2770ecde5a0a2ad59da1e58649477be2bd1237843b729004113d447bdeba9c359f0d36
+  data.tar.gz: 15087aa4dbd3365c88b3ab75cbf02de30cd9e0250bf3b372aeee155b2155239db68b75934d920120771e126b14825d7e4544bbca0bf5fb2dff3f483a6837eee1

data/ChangeLog

@@ -1,3 +1,6 @@
+Release 0.2.0 - 2016-06-03
+* Support path option to allow to specify list of target objects directly @sonots thanks! [#17](https://github.com/embulk/embulk-input-gcs/pull/17)
+
 Release 0.1.13 - 2016-02-04
 * Upgraded embulk to v0.8.2
 * Updated Google HTTP Client Library from 1.19.0 to 2.1.21.0

data/README.md

@@ -37,7 +37,8 @@ embulk run /path/to/config.yml
 ## Configuration
 
 - **bucket** Google Cloud Storage bucket name (string, required)
-- **path_prefix** prefix of target keys (string, required)
+- **path_prefix** prefix of target keys (string, either of "path_prefix" or "paths" is required)
+- **paths** list of target keys (array of string, either of "path_prefix" or "paths" is required)
 - **auth_method**  (string, optional, "private_key", "json_key" or "compute_engine". default value is "private_key")
 - **service_account_email** Google Cloud Storage service_account_email (string, required when auth_method is private_key)
 - **p12_keyfile** fullpath of p12 key (string, required when auth_method is private_key)
@@ -147,6 +148,13 @@ in:
   auth_method: compute_engine
 ```
 
+## Eventually Consistency
+
+An operation listing objects is eventually consistent although getting objects is strongly consistent, see https://cloud.google.com/storage/docs/consistency.
+
+`path_prefix` uses the objects list API, therefore it would miss some of objects.
+If you want to avoid such situations, you should use `paths` option which directly specifies object paths without the objects list API.
+
 ## Build
 
 ```
@@ -201,4 +209,4 @@ $ launchctl load ~/Library/LaunchAgents/environment.plist
 $ launchctl getenv GCP_EMAIL //try to get value.
 
 Then start your applications.
-```
+```

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.13"
+version = "0.2.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -52,7 +52,8 @@ public class GcsFileInputPlugin
         String getBucket();
 
         @Config("path_prefix")
-        String getPathPrefix();
+        @ConfigDefault("null")
+        Optional<String> getPathPrefix();
 
         @Config("last_path")
         @ConfigDefault("null")
@@ -84,6 +85,8 @@ public class GcsFileInputPlugin
         @ConfigDefault("null")
         Optional<LocalFile> getJsonKeyfile();
 
+        @Config("paths")
+        @ConfigDefault("[]")
         List<String> getFiles();
         void setFiles(List<String> files);
 
@@ -131,8 +134,13 @@ public class GcsFileInputPlugin
 
         Storage client = newGcsClient(task, newGcsAuth(task));
 
-        // list files recursively
-        task.setFiles(listFiles(task, client));
+        // list files recursively if path_prefix is specified
+        if (task.getPathPrefix().isPresent()) {
+            task.setFiles(listFiles(task, client));
+        }
+        if (task.getFiles().isEmpty()) {
+            throw new ConfigException("No file is found. Fix path_prefix or specify paths directly");
+        }
         // number of processors is same with number of files
         return resume(task.dump(), task.getFiles().size(), control);
     }
@@ -214,7 +222,7 @@ public class GcsFileInputPlugin
     {
         String bucket = task.getBucket();
 
-        return listGcsFilesByPrefix(client, bucket, task.getPathPrefix(), task.getLastPath());
+        return listGcsFilesByPrefix(client, bucket, task.getPathPrefix().get(), task.getLastPath());
     }
 
     /**

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -36,6 +36,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assume.assumeNotNull;
 
 import java.lang.reflect.InvocationTargetException;
@@ -107,6 +108,38 @@ public class TestGcsFileInputPlugin
         assertEquals("Embulk GCS input plugin", task.getApplicationName());
     }
 
+    // paths are set
+    @Test
+    public void checkDefaultValuesPathsSpecified()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("paths", Arrays.asList("object1", "object2"))
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("p12_keyfile_fullpath", GCP_P12_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
+
+        GcsFileInputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
+        assertFalse(task.getFiles().isEmpty());
+    }
+
+    // both path_prefix and paths are not set
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesNoPathSpecified()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("p12_keyfile_fullpath", GCP_P12_KEYFILE)
+                .set("parser", parserConfig(schemaConfig()));
+
+        runner.transaction(config, new Control());
+    }
+
     // p12_keyfile is null when auth_method is private_key
     @Test(expected = ConfigException.class)
     public void checkDefaultValuesP12keyNull()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.2.0
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-04 00:00:00.000000000 Z
+date: 2016-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,7 +65,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.13.jar
+- classpath/embulk-input-gcs-0.2.0.jar
 - classpath/google-api-client-1.21.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar