embulk-input-gcs 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d25891998dc1d0794b0fcfac04ea7a011f777f9c
-  data.tar.gz: cd7d9977e6bfa8df5212181ba49aaacf7998110b
+  metadata.gz: fd9bbc407e6e93967a7d08fcd72b4d4f0c419659
+  data.tar.gz: 86ebeaf4dace9de06146a4671d6fe3db69089b5c
 SHA512:
-  metadata.gz: 9beb8bfab93341c05b076fb1937096cb86ab850da685806cd1aa29cf65ffea356e318bfdddc566321499e4ffa8b5ccac9c8028b0dd83aa20f29a6955c8556e9a
-  data.tar.gz: d3f872c0dd1460e5234b836761fbd7ea5f723b17371d881e4a91989b62edd10cfa98a0110eb26af5940c684a318164b1448f0ccd73962bc5db126b5c60070e8b
+  metadata.gz: 783728ec908c8f598acfac3ec555075a55b7249943afc76d030f2fe57d9353ddbb78cbecabdba9504189d943b9574555d3ba301a4ee654f5c78b81c179abdef4
+  data.tar.gz: 0e8815c728a060f749e8d797938235b72eed889c2155b635256646e5816432326555648cf229ef76a01e969aca6ec659eb5ce976c43f32ee64f6a3d3ce451d69

data/ChangeLog

@@ -1,6 +1,11 @@
+Release 0.1.10 - 2015-11-07
+
+* Fix resume download logics
+* Throw ConfigException when files listing failed. @muga thanks!
+
 Release 0.1.9 - 2015-10-30
 
-Fix GcsAuthentication object initialization for mapreduce executor. @muga thanks!
+* Fix GcsAuthentication object initialization for mapreduce executor. @muga thanks!
 
 Release 0.1.8 - 2015-10-29
 
@@ -27,4 +32,4 @@ Release 0.1.4 - 2015-06-27
 
 Release 0.1.3 - 2015-03-16
 
-* Changed supported Java version from 8 to 7
+* Changed supported Java version from 8 to 7

data/build.gradle

@@ -16,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.9"
+version = "0.1.10"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.5"

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -7,10 +7,13 @@ import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
 
+import com.google.api.client.http.HttpResponseException;
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
+import com.google.common.base.Charsets;
+import com.google.common.io.BaseEncoding;
 import java.security.GeneralSecurityException;
 
 import org.embulk.config.TaskReport;
@@ -114,6 +117,13 @@ public class GcsFileInputPlugin
             }
         }
 
+        // @see https://cloud.google.com/storage/docs/bucket-naming
+        if (task.getLastPath().isPresent()) {
+            if (task.getLastPath().get().length() >= 128) {
+                throw new ConfigException("last_path length is allowed between 1 and 1024 bytes");
+            }
+        }
+
         Storage client = newGcsClient(task, newGcsAuth(task));
 
         // list files recursively
@@ -209,7 +219,7 @@ public class GcsFileInputPlugin
     {
         ImmutableList.Builder<String> builder = ImmutableList.builder();
 
-        String lastKey = lastPath.orNull();
+        String lastKey = lastPath.isPresent() ? base64Encode(lastPath.get()) : null;
 
         // @see https://cloud.google.com/storage/docs/json_api/v1/objects#resource
         try {
@@ -250,6 +260,10 @@ public class GcsFileInputPlugin
                 listObjects.setPageToken(lastKey);
             } while (lastKey != null);
         } catch (IOException e) {
+            if ((e instanceof HttpResponseException) && ((HttpResponseException) e).getStatusCode() == 400) {
+                throw new ConfigException(String.format("Files listing failed: bucket:%s, prefix:%s, last_path:%s", bucket, prefix, lastKey), e);
+            }
+
             log.warn(String.format("Could not get file list from bucket:%s", bucket));
             log.warn(e.getMessage());
         }
@@ -315,6 +329,24 @@ public class GcsFileInputPlugin
         public void close() { }
     }
 
+    // String nextToken = base64Encode(0x0a + 0x01~0x27 + filePath);
+    private static String base64Encode(String path)
+    {
+        byte[] encoding;
+        byte[] utf8 = path.getBytes(Charsets.UTF_8);
+        log.debug(String.format("path string: %s ,path length:%s \" + ", path, utf8.length));
+
+        encoding = new byte[utf8.length + 2];
+        encoding[0] = 0x0a;
+        encoding[1] = new Byte(String.valueOf(path.length()));
+        System.arraycopy(utf8, 0, encoding, 2, utf8.length);
+
+        String s = BaseEncoding.base64().encode(encoding);
+        log.debug(String.format("last_path(base64 encoded): %s" ,s));
+        return s;
+    }
+
+
     public enum AuthMethod
     {
         private_key("private_key"),

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -157,6 +157,22 @@ public class TestGcsFileInputPlugin
         runner.transaction(config, new Control());
     }
 
+    // last_path length is too long
+    @Test(expected = ConfigException.class)
+    public void checkDefaultValuesLongLastPath()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("path_prefix", "my-prefix")
+                .set("auth_method", "json_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("json_keyfile", null)
+                .set("last_path", "ccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc128")
+                .set("parser", parserConfig(schemaConfig()));
+
+        runner.transaction(config, new Control());
+    }
+
     @Test
     public void testGcsClientCreateSuccessfully()
             throws GeneralSecurityException, IOException, NoSuchMethodException,
@@ -276,6 +292,22 @@ public class TestGcsFileInputPlugin
         assertRecords(config, output);
     }
 
+    @Test
+    public void testBase64()
+            throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
+    {
+        Method method = GcsFileInputPlugin.class.getDeclaredMethod("base64Encode", String.class);
+        method.setAccessible(true);
+
+        assertEquals("CgFj", method.invoke(plugin, "c"));
+        assertEquals("CgJjMg==", method.invoke(plugin, "c2"));
+        assertEquals("Cgh0ZXN0LmNzdg==", method.invoke(plugin, "test.csv"));
+        assertEquals("ChZnY3MtdGVzdC9zYW1wbGVfMDEuY3N2", method.invoke(plugin, "gcs-test/sample_01.csv"));
+        String params = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc127";
+        String expected = "Cn9jY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjMTI3";
+        assertEquals(expected, method.invoke(plugin, params));
+    }
+
     public ConfigSource config()
     {
         return Exec.newConfigSource()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-10-30 00:00:00.000000000 Z
+date: 2015-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,7 +63,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.9.jar
+- classpath/embulk-input-gcs-0.1.10.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar