embulk-input-gcs 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9113cbc31b8a47e6dbb6377c3ae7dcbb3d4f03e5
-  data.tar.gz: 60744784255a366230bce11a16632b42df6c08f2
+  metadata.gz: 206cae7d3839c48e0d5463c12143a602de77d3fa
+  data.tar.gz: 2d7215927a470c9ba7540970057b81365a975dd9
 SHA512:
-  metadata.gz: 8e1f54da44d5ae5cc84815a76a3a13c9d289be3844ff322c49f3463ac918140193a9eef35d21b319b792c4149c5c03498dbbf0d9ca7123d782608971aaa40251
-  data.tar.gz: f66b8c1bc9d6d0ed8663b19edfbb815850cf2dd24bb3fc60e1c7cb20a4381a1967d4aef35a05ab3633717d3fb16da616cbf70687e0f74f354fb1b992e1dd55f2
+  metadata.gz: d98845cbb8c6d71a464430c6758674fcbb68af5d632723be2289b07f3b69991b0074a1dbbeaa445fa7e4a1a171aa9a3e5b962df4670fb88ec32883348d5eb6f9
+  data.tar.gz: d499db32c291a9abc349ea3b0969fb32eea164f06723323167aa9529cd9c2abfbf6135a275439da3d8597687525ff568f7133864674bd553d039be92cf58e407

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.3 - 2016-08-09
+* [new feature] Support `incremental` option [#24](https://github.com/embulk/embulk-input-gcs/pull/24)
+
 ## 0.2.2 - 2016-08-08
 * [maintenance] Fix validation logic for `path_prefix` and `paths` options [#23](https://github.com/embulk/embulk-input-gcs/pull/23)
 

data/README.md

@@ -40,6 +40,7 @@ embulk run /path/to/config.yml
 - **bucket** Google Cloud Storage bucket name (string, required)
 - **path_prefix** prefix of target keys (string, either of "path_prefix" or "paths" is required)
 - **paths** list of target keys (array of string, either of "path_prefix" or "paths" is required)
+- **incremental**: enables incremental loading(boolean, optional. default: true. If incremental loading is enabled, config diff for the next execution will include `last_path` parameter so that next execution skips files before the path. Otherwise, `last_path` will not be included.
 - **auth_method**  (string, optional, "private_key", "json_key" or "compute_engine". default value is "private_key")
 - **service_account_email** Google Cloud Storage service_account_email (string, required when auth_method is private_key)
 - **p12_keyfile** fullpath of p12 key (string, required when auth_method is private_key)

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.2"
+version = "0.2.3"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.2"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -59,6 +59,10 @@ public class GcsFileInputPlugin
         @ConfigDefault("null")
         Optional<String> getLastPath();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        boolean getIncremental();
+
         @Config("auth_method")
         @ConfigDefault("\"private_key\"")
         AuthMethod getAuthMethod();
@@ -179,15 +183,17 @@ public class GcsFileInputPlugin
         ConfigDiff configDiff = Exec.newConfigDiff();
 
         List<String> files = new ArrayList<String>(task.getFiles());
-        if (files.isEmpty()) {
-            // keep the last value if any
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
+        if (task.getIncremental()) {
+            if (files.isEmpty()) {
+                // keep the last value if any
+                if (task.getLastPath().isPresent()) {
+                    configDiff.set("last_path", task.getLastPath().get());
+                }
+            }
+            else {
+                Collections.sort(files);
+                configDiff.set("last_path", files.get(files.size() - 1));
             }
-        }
-        else {
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
         }
 
         return configDiff;

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -105,6 +105,7 @@ public class TestGcsFileInputPlugin
                 .set("path_prefix", "my-prefix");
 
         GcsFileInputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
+        assertEquals(true, task.getIncremental());
         assertEquals("private_key", task.getAuthMethod().toString());
         assertEquals("Embulk GCS input plugin", task.getApplicationName());
     }
@@ -302,6 +303,17 @@ public class TestGcsFileInputPlugin
         assertEquals(GCP_BUCKET_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
     }
 
+    @Test
+    public void testListFilesByPrefixIncrementalFalse() throws Exception
+    {
+        ConfigSource config = config().deepCopy()
+                .set("incremental", false);
+
+        ConfigDiff configDiff = runner.transaction(config, new Control());
+
+        assertEquals("{}", configDiff.toString());
+    }
+
     @Test
     public void testListFilesByPrefixNonExistsBucket()
             throws NoSuchMethodException, IllegalAccessException, InvocationTargetException

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-08 00:00:00.000000000 Z
+date: 2016-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ files:
 - src/test/resources/secretkeys.tar.enc
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.2.2.jar
+- classpath/embulk-input-gcs-0.2.3.jar
 - classpath/google-api-client-1.21.0.jar
 - classpath/google-api-services-storage-v1-rev59-1.21.0.jar
 - classpath/google-http-client-1.21.0.jar