embulk-output-gcs 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5f66c9aebb242064e49a624cb08443142d7440b
-  data.tar.gz: bb3ef12c25a3bb768e2453012e351073566bf5b1
+  metadata.gz: b41fdc69131fe0099ff844875bafa6d160510e10
+  data.tar.gz: 5b084d56f12509c0c273ea295519a1534d310579
 SHA512:
-  metadata.gz: 337c8a5c9cbf80008865a89dae91bad27de12915354b6937223ce72ef9f4295d089495ee577b2e29001486bb9cc1ff4217b23d1a7d7832bedaac9228ac3f224c
-  data.tar.gz: e26a507cebd8e93f87b0b5cd8b7c206901cc3ff9ca01b85a92e4ec721a2c84e164a009b134c259fb88a5e35ef8f02b47aaf2ebb9eab22d45948acb30cc2dadc4
+  metadata.gz: a7f51eec122063edaffe94aa065f9bdd1dcc09752d46296f3f8d276d5af250ef82f1a2f56b7e0163f646c98731954d4bf5ce2b901f4e22b4e09033fd20fbf627
+  data.tar.gz: 7ed822140abca79f035a9bf59af3d472472b6bdfa4c1db505a22ae4ca0a80ca1fd380635487f72eb8fd74f706fd1e531fdf421f63b1614f6fcc5731b319c1489

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.4.0"
+version = "0.4.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.6"

data/src/main/java/org/embulk/output/GcsOutputPlugin.java

@@ -255,7 +255,7 @@ public class GcsOutputPlugin implements FileOutputPlugin
         @Override
         public void finish()
         {
-            String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
+            String path = generateRemotePath(pathPrefix, sequenceFormat, taskIndex, fileIndex, pathSuffix);
             close();
             if (tempFile != null) {
                 currentUpload = startUpload(path);
@@ -419,6 +419,19 @@ public class GcsOutputPlugin implements FileOutputPlugin
         }
     }
 
+    /**
+     * GCS has character limitation in object names.
+     * @see https://cloud.google.com/storage/docs/naming#objectnames
+     * Although "." isn't listed at above pages, we can't access "./" path from GUI console.
+     * And in many cases, user don't intend of creating "/" directory under the bucket.
+     * This method normalizes path when it contains "./" and "/" and its variations at the beginning
+     */
+    private static String generateRemotePath(String pathPrefix, String sequenceFormat, int taskIndex, int fileIndex, String pathSuffix)
+    {
+        String path = pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + pathSuffix;
+        return path.replaceFirst("^\\.*/*", "");
+    }
+
     public enum AuthMethod
     {
         private_key("private_key"),

data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java

@@ -70,7 +70,7 @@ public class TestGcsOutputPlugin
         assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
 
         GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
-        GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
+        GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "output_";
         LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
         GCP_APPLICATION_NAME = "embulk-output-gcs";
     }
@@ -275,10 +275,28 @@ public class TestGcsOutputPlugin
         output.finish();
         output.commit();
 
-        String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
+        String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 1) + task.getFileNameExtension();
         assertRecords(remotePath);
     }
 
+    @Test
+    public void testGenerateRemotePath() throws Exception
+    {
+        ConfigSource configSource = config();
+        PluginTask task = configSource.loadConfig(PluginTask.class);
+        Method method = GcsOutputPlugin.class.getDeclaredMethod("generateRemotePath", String.class, String.class, int.class, int.class, String.class);
+        method.setAccessible(true);
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "/sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "./sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "../sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "//sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("path/to/sample.000.01.csv", method.invoke(plugin, "/path/to/sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("path/to/./sample.000.01.csv", method.invoke(plugin, "path/to/./sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("path/to/../sample.000.01.csv", method.invoke(plugin, "path/to/../sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "....../sample", task.getSequenceFormat(), 0, 1, ".csv"));
+        assertEquals("sample.000.01.csv", method.invoke(plugin, "......///sample", task.getSequenceFormat(), 0, 1, ".csv"));
+    }
+
     public ConfigSource config()
     {
         return Exec.newConfigSource()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-gcs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-22 00:00:00.000000000 Z
+date: 2017-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ files:
 - src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-gcs-0.4.0.jar
+- classpath/embulk-output-gcs-0.4.1.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev28-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar