embulk-input-gcs 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: dfc015455f0851ec196018a6ede5835653ef7f5a
-  data.tar.gz: 47185bd68f3bbe58070094fd9d87bda3635476e4
+SHA256:
+  metadata.gz: 1c252ffc4a609f7e29fffc86eb665c1032e2580b04277671157d44450becf3c0
+  data.tar.gz: a5c8ce1851371760195c3f51329fd3eb20f3e6d34e643d8ff833d18d77205b8c
 SHA512:
-  metadata.gz: ef1608fd099a302e4977b3808d7adb39ea2718937b893545f147167579b9d0ff0dab8b231ab033650c340f0c66588f4fb4568704ea19af499979f0cb7c51276a
-  data.tar.gz: e1a57cf92f6c81bb79bd7476d08eaecfab9c9d8c21f61f5d5b32e680cf70b0c9503c8bbc2ff7f2331f50a073c60010a55d105c37c0559cf66c82c439c47d7d5a
+  metadata.gz: e46064fe213ac097602e355f8d2e456ad16d070ada11ed7fc0e5f50c571f20514b2a97eb59cb9ce2c31d9a2d6f39cb05ef76edd5c65c2e27de60d2e39491decc
+  data.tar.gz: 468afcb058f36ab42e40f8267450156f5fd09b3f54ee5ee17e7f7b23ac976a5dd19322cb624b6d25bf4de606f37d1738d6cceeb927ebd42983519295af7f53f2

data/.travis.yml

@@ -5,16 +5,16 @@ jdk:
 
 before_install:
   - openssl aes-256-cbc -K $encrypted_407f38382059_key -iv $encrypted_407f38382059_iv
-    -in src/test/resources/secretkeys.tar.enc -out ./secretkeys.tar -d
-  - tar xvf secretkeys.tar
+    -in src/test/resources/secrets.tar.enc -out ./secrets.tar -d
+  - tar xvf secrets.tar
 
 env:
   global:
     - GCP_EMAIL=unittest@embulk-input-gcs-test.iam.gserviceaccount.com
     - GCP_BUCKET=embulk-input-gcs-test
     - GCP_BUCKET_DIRECTORY=unittests_import
-    - GCP_P12_KEYFILE=./embulk-input-gcs-test-9d51de6767a8.p12
-    - GCP_JSON_KEYFILE=./embulk-input-gcs-test-65cfe930dca1.json
+    - GCP_P12_KEYFILE=./embulk-input-gcs-test-40bbfa15acd2.p12
+    - GCP_JSON_KEYFILE=./embulk-input-gcs-test-2492bb6e0f66.json
 
 script:
   - ./gradlew gem

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.3.1 - 2019-01-07
+* [maintenance] Updated Google Cloud Storage API Client Library from v1-rev59-1.21.0 to 1.56.0 [#40](https://github.com/embulk/embulk-input-gcs/pull/40)
+* [new feature] Refactor to use InputStream (to replace local file download)
+
 ## 0.3.0 - 2018-12-26
 * [maintenance] Use file name output feature in Embulk core to show file name in cmdout [#37](https://github.com/embulk/embulk-input-gcs/pull/37)
 

data/build.gradle

@@ -1,11 +1,14 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
-    id "com.github.jruby-gradle.base" version "0.1.5"
+    id "com.jfrog.bintray" version "1.8.4"
+    id "com.github.jruby-gradle.base" version "1.6.0"
+    id "com.github.johnrengelman.shadow" version "4.0.3"
+    id "com.github.spotbugs" version "1.6.8"
     id "java"
     id "checkstyle"
     id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
+
 repositories {
     mavenCentral()
     jcenter()
@@ -17,38 +20,61 @@ configurations {
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-version = "0.3.0"
+version = "0.3.1"
+
+// Relocate Guava packages since it's incompatible with Guava's version from Embulk
+shadowJar {
+    classifier = "shadow"
+    exclude "org/embulk/plugin/**"
+
+    dependencies {
+        include dependency("com.google.guava:guava")
+        include dependency("com.google.cloud:google-cloud-storage")
+    }
+
+    relocate "com.google.common", "embulk.gcs.com.google.common"
+}
 
 dependencies {
     compile  "org.embulk:embulk-core:0.9.12"
     provided "org.embulk:embulk-core:0.9.12"
 
-    compile "com.google.http-client:google-http-client-jackson2:1.21.0"
-    compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
+    compile "com.google.cloud:google-cloud-storage:1.56.0"
 
     testCompile "junit:junit:4.12"
     testCompile "org.embulk:embulk-core:0.9.12:tests"
     testCompile "org.embulk:embulk-standards:0.9.12"
+    testCompile "org.mockito:mockito-core:2.18.3"
 }
 
-task classpath(type: Copy, dependsOn: ["jar"]) {
+task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
     doFirst { file("classpath").deleteDir() }
-    from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    from (configurations.runtime - configurations.provided
+            + configurations.shadow
+            - files(shadowJar.getIncludedDependencies())
+            + files(shadowJar.archivePath))
     into "classpath"
 }
 clean { delete 'classpath' }
 
+test {
+    jvmArgs '-XX:MaxPermSize=128M', '-Xmx2048m'
+    testLogging {
+        events "passed", "skipped", "failed"
+    }
+}
+
 checkstyle {
     configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
     toolVersion = '6.14.1'
 }
 checkstyleMain {
     configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-    ignoreFailures = true
+    ignoreFailures = false
 }
 checkstyleTest {
     configFile = file("${project.rootDir}/config/checkstyle/default.xml")
-    ignoreFailures = true
+    ignoreFailures = false
 }
 task checkstyle(type: Checkstyle) {
     classpath = sourceSets.main.output + sourceSets.test.output
@@ -56,12 +82,14 @@ task checkstyle(type: Checkstyle) {
 }
 
 task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
-    jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
-    script "build/gemspec"
+    script "gem"
+    scriptArgs "build", "build/gemspec"
     doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
 }
 
-task gemspec << { file("build/gemspec").write($/
+task gemspec {
+    doLast {
+        file("build/gemspec").write($/
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
@@ -80,4 +108,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rake', ['>= 10.0']
 end
 /$)
+    }
 }

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sun Jan 08 00:35:58 PST 2017
+#Tue Dec 18 15:27:07 ICT 2018
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -6,12 +6,30 @@
 ##
 ##############################################################################
 
-# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+# Attempt to set APP_HOME
+# Resolve links: $0 may be a link
+PRG="$0"
+# Need this for relative symlinks.
+while [ -h "$PRG" ] ; do
+    ls=`ls -ld "$PRG"`
+    link=`expr "$ls" : '.*-> \(.*\)$'`
+    if expr "$link" : '/.*' > /dev/null; then
+        PRG="$link"
+    else
+        PRG=`dirname "$PRG"`"/$link"
+    fi
+done
+SAVED="`pwd`"
+cd "`dirname \"$PRG\"`/" >/dev/null
+APP_HOME="`pwd -P`"
+cd "$SAVED" >/dev/null
 
 APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS=""
+
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"
 
@@ -30,6 +48,7 @@ die ( ) {
 cygwin=false
 msys=false
 darwin=false
+nonstop=false
 case "`uname`" in
   CYGWIN* )
     cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
   MINGW* )
     msys=true
     ;;
+  NONSTOP* )
+    nonstop=true
+    ;;
 esac
 
-# For Cygwin, ensure paths are in UNIX format before anything is touched.
-if $cygwin ; then
-    [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
-fi
-
-# Attempt to set APP_HOME
-# Resolve links: $0 may be a link
-PRG="$0"
-# Need this for relative symlinks.
-while [ -h "$PRG" ] ; do
-    ls=`ls -ld "$PRG"`
-    link=`expr "$ls" : '.*-> \(.*\)$'`
-    if expr "$link" : '/.*' > /dev/null; then
-        PRG="$link"
-    else
-        PRG=`dirname "$PRG"`"/$link"
-    fi
-done
-SAVED="`pwd`"
-cd "`dirname \"$PRG\"`/" >&-
-APP_HOME="`pwd -P`"
-cd "$SAVED" >&-
-
 CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
 
 # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
 fi
 
 # Increase the maximum file descriptors if we can.
-if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
+if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
     MAX_FD_LIMIT=`ulimit -H -n`
     if [ $? -eq 0 ] ; then
         if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
 if $cygwin ; then
     APP_HOME=`cygpath --path --mixed "$APP_HOME"`
     CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
+    JAVACMD=`cygpath --unix "$JAVACMD"`
 
     # We build the pattern for arguments to be converted via cygpath
     ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
+
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -8,14 +8,14 @@
 @rem Set local scope for the variables with windows NT shell
 if "%OS%"=="Windows_NT" setlocal
 
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
 set DIRNAME=%~dp0
 if "%DIRNAME%" == "" set DIRNAME=.
 set APP_BASE_NAME=%~n0
 set APP_HOME=%DIRNAME%
 
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
 @rem Find java.exe
 if defined JAVA_HOME goto findJavaFromJavaHome
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
 goto fail
 
 :init
-@rem Get command-line arguments, handling Windowz variants
+@rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/src/main/java/org/embulk/input/gcs/AuthUtils.java

@@ -0,0 +1,108 @@
+package org.embulk.input.gcs;
+
+import com.google.api.client.util.SecurityUtils;
+import com.google.api.services.storage.StorageScopes;
+import com.google.auth.Credentials;
+import com.google.auth.oauth2.ServiceAccountCredentials;
+import com.google.cloud.storage.Storage;
+import com.google.cloud.storage.StorageException;
+import com.google.cloud.storage.StorageOptions;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.unit.LocalFile;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.GeneralSecurityException;
+import java.security.PrivateKey;
+import java.util.Optional;
+
+class AuthUtils
+{
+    public enum AuthMethod
+    {
+        private_key, compute_engine, json_key
+    }
+
+    interface Task
+    {
+        @Config("auth_method")
+        @ConfigDefault("\"private_key\"")
+        AuthUtils.AuthMethod getAuthMethod();
+
+        @Config("service_account_email")
+        @ConfigDefault("null")
+        Optional<String> getServiceAccountEmail();
+
+        // kept for backward compatibility
+        @Config("p12_keyfile_fullpath")
+        @ConfigDefault("null")
+        Optional<String> getP12KeyfileFullpath();
+
+        @Config("p12_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getP12Keyfile();
+        void setP12Keyfile(Optional<LocalFile> p12Keyfile);
+
+        @Config("json_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getJsonKeyfile();
+    }
+
+    private AuthUtils()
+    {
+    }
+
+    static Storage newClient(final PluginTask task)
+    {
+        try {
+            final StorageOptions.Builder builder = StorageOptions.newBuilder();
+            switch (task.getAuthMethod()) {
+                case json_key:
+                    builder.setCredentials(fromJson(task));
+                    break;
+                case private_key:
+                    builder.setCredentials(fromP12(task));
+                    break;
+                default:
+                    // compute_engine does not need credentials
+                    break;
+            }
+            // test client to verify auth
+            final Storage client = builder.build().getService();
+            client.list(task.getBucket(), Storage.BlobListOption.pageSize(1));
+            return client;
+        }
+        catch (StorageException | IOException | GeneralSecurityException e) {
+            throw new ConfigException(e);
+        }
+    }
+
+    @VisibleForTesting
+    static Credentials fromP12(final Task task) throws IOException, GeneralSecurityException
+    {
+        final String path = task.getP12Keyfile().get().getPath().toString();
+        try (final InputStream p12InputStream = new FileInputStream(path)) {
+            final PrivateKey pk = SecurityUtils.loadPrivateKeyFromKeyStore(
+                    SecurityUtils.getPkcs12KeyStore(), p12InputStream, "notasecret",
+                    "privatekey", "notasecret");
+            return ServiceAccountCredentials.newBuilder()
+                    .setClientEmail(task.getServiceAccountEmail().orElse(null))
+                    .setPrivateKey(pk)
+                    .setScopes(ImmutableList.of(StorageScopes.DEVSTORAGE_READ_ONLY))
+                    .build();
+        }
+    }
+
+    @VisibleForTesting
+    static Credentials fromJson(final Task task) throws IOException
+    {
+        final String path = task.getJsonKeyfile().map(f -> f.getPath().toString()).get();
+        final InputStream jsonStream = new FileInputStream(path);
+        return ServiceAccountCredentials.fromStream(jsonStream);
+    }
+}

data/src/main/java/org/embulk/input/gcs/FileList.java

@@ -3,6 +3,7 @@ package org.embulk.input.gcs;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Preconditions;
 import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -122,7 +123,7 @@ public class FileList
             return this;
         }
 
-        public Builder pathMatchPattern(String pattern)
+        public synchronized Builder pathMatchPattern(String pattern)
         {
             this.pathMatchPattern = Pattern.compile(pattern);
             return this;
@@ -168,7 +169,7 @@ public class FileList
             return true;
         }
 
-        public FileList build()
+        public synchronized FileList build()
         {
             try {
                 stream.close();
@@ -211,7 +212,7 @@ public class FileList
             @JsonProperty("tasks") List<List<Entry>> tasks,
             @JsonProperty("last") Optional<String> last)
     {
-        this.data = data;
+        this.data = data.clone();
         this.tasks = tasks;
         this.last = last;
     }
@@ -241,7 +242,7 @@ public class FileList
     @Deprecated
     public byte[] getData()
     {
-        return data;
+        return data.clone();
     }
 
     @JsonProperty("tasks")
@@ -258,7 +259,7 @@ public class FileList
         return last;
     }
 
-    private class EntryList
+    private static class EntryList
             extends AbstractList<String>
     {
         private final byte[] data;
@@ -313,10 +314,12 @@ public class FileList
         private byte[] readNext()
         {
             try {
-                stream.read(castBuffer.array());
-                int n = castBuffer.getInt(0);
-                byte[] b = new byte[n];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
-                stream.read(b);
+                int n = stream.read(castBuffer.array());
+                Preconditions.checkArgument(n == castBuffer.capacity(), "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
+                int len = castBuffer.getInt(0);
+                byte[] b = new byte[len];  // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
+                n = stream.read(b);
+                Preconditions.checkArgument(n == len, "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
 
                 current++;