embulk-input-gcs 0.3.1 → 0.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1c252ffc4a609f7e29fffc86eb665c1032e2580b04277671157d44450becf3c0
-  data.tar.gz: a5c8ce1851371760195c3f51329fd3eb20f3e6d34e643d8ff833d18d77205b8c
+  metadata.gz: 1464dc7e445f4ed06ee28336fb0eb53534fbb78b7ee6272c25d9b8a7faece5f2
+  data.tar.gz: b7f87e75f2bc970e9e4aa890299528b6a930f3e621444495f5563561c07a81c6
 SHA512:
-  metadata.gz: e46064fe213ac097602e355f8d2e456ad16d070ada11ed7fc0e5f50c571f20514b2a97eb59cb9ce2c31d9a2d6f39cb05ef76edd5c65c2e27de60d2e39491decc
-  data.tar.gz: 468afcb058f36ab42e40f8267450156f5fd09b3f54ee5ee17e7f7b23ac976a5dd19322cb624b6d25bf4de606f37d1738d6cceeb927ebd42983519295af7f53f2
+  metadata.gz: dbff12cfba5f647596023f6185f4177a533b0e90b738ce9861d437fc3304a971da471e8c06bd4564174cfd16f709488de76a65aba2a75ead3a78932663d1a3c3
+  data.tar.gz: da85294930e6cf321164d750f228ee3bafa91a352ed0fbfd44673509ef58acf70f629b98523eea7a38a8dd110b58cca2ba6542fc30ad0704c97264573bf42692

data/.travis.yml

@@ -1,5 +1,7 @@
 language: java
 
+dist: trusty
+
 jdk:
   - oraclejdk8
 

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.2 - 2020-01-13
+* [maintenance] Improve base64encode method for last_path [#47](https://github.com/embulk/embulk-input-gcs/pull/47)
+
 ## 0.3.1 - 2019-01-07
 * [maintenance] Updated Google Cloud Storage API Client Library from v1-rev59-1.21.0 to 1.56.0 [#40](https://github.com/embulk/embulk-input-gcs/pull/40)
 * [new feature] Refactor to use InputStream (to replace local file download)

data/build.gradle

@@ -20,7 +20,7 @@ configurations {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-version = "0.3.1"
+version = "0.3.2"
 
 // Relocate Guava packages since it's incompatible with Guava's version from Embulk
 shadowJar {
@@ -81,7 +81,7 @@ task checkstyle(type: Checkstyle) {
     source = sourceSets.main.allJava + sourceSets.test.allJava
 }
 
-task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
+task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
     script "gem"
     scriptArgs "build", "build/gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }

data/src/main/java/org/embulk/input/gcs/GcsFileInput.java

@@ -80,7 +80,7 @@ public class GcsFileInput
         return builder.build();
     }
 
-    // String nextToken = base64Encode(0x0a + 0x01~0x27 + filePath);
+    // String nextToken = base64Encode(0x0a + ASCII character according to utf8EncodeLength position+ filePath);
     @VisibleForTesting
     static String base64Encode(String path)
     {
@@ -88,9 +88,17 @@ public class GcsFileInput
         byte[] utf8 = path.getBytes(Charsets.UTF_8);
         LOG.debug("path string: {} ,path length:{} \" + ", path, utf8.length);
 
+        int utf8EncodeLength = utf8.length;
+        if (utf8EncodeLength >= 128) {
+            throw new ConfigException(String.format("last_path '%s' is too long to encode. Please try to reduce its length", path));
+        }
+
         encoding = new byte[utf8.length + 2];
         encoding[0] = 0x0a;
-        encoding[1] = Byte.valueOf(String.valueOf(path.length()));
+
+        // for example: 60 -> '<'
+        char temp = (char) utf8EncodeLength;
+        encoding[1] = (byte) temp;
         System.arraycopy(utf8, 0, encoding, 2, utf8.length);
 
         String s = BaseEncoding.base64().encode(encoding);

data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java

@@ -346,6 +346,23 @@ public class TestGcsFileInputPlugin
         runner.transaction(config, new Control());
     }
 
+    @Test(expected = ConfigException.class)
+    public void testLastPathTooLong() throws Exception
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("bucket", GCP_BUCKET)
+                .set("paths", Arrays.asList())
+                .set("auth_method", "private_key")
+                .set("service_account_email", GCP_EMAIL)
+                .set("p12_keyfile", GCP_P12_KEYFILE)
+                .set("json_keyfile", GCP_JSON_KEYFILE)
+                .set("application_name", GCP_APPLICATION_NAME)
+                .set("last_path", "テストダミー/テストダミーテストダミーテストダミーテストダミーテストダミーテストダミーテストダミー.csv")
+                .set("parser", parserConfig(schemaConfig()));
+
+        runner.transaction(config, new Control());
+    }
+
     @Test
     public void testGcsFileInputByOpen()
     {
@@ -375,6 +392,10 @@ public class TestGcsFileInputPlugin
         String params = "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc127";
         String expected = "Cn9jY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjY2NjMTI3";
         assertEquals(expected, GcsFileInput.base64Encode(params));
+
+        params = "テストダミー/テス123/テストダミー/テストダミ.csv";
+        expected = "CkPjg4bjgrnjg4jjg4Djg5/jg7wv44OG44K5MTIzL+ODhuOCueODiOODgOODn+ODvC/jg4bjgrnjg4jjg4Djg58uY3N2";
+        assertEquals(expected, GcsFileInput.base64Encode(params));
     }
 
     private ConfigSource config()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-07 00:00:00.000000000 Z
+date: 2020-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -56,7 +56,7 @@ files:
 - classpath/checker-compat-qual-2.5.2.jar
 - classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-gcs-0.3.1-shadow.jar
+- classpath/embulk-input-gcs-0.3.2-shadow.jar
 - classpath/error_prone_annotations-2.1.3.jar
 - classpath/gax-1.35.1.jar
 - classpath/gax-httpjson-0.52.1.jar