embulk-input-gcs 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
- SHA1:
3
- metadata.gz: dfc015455f0851ec196018a6ede5835653ef7f5a
4
- data.tar.gz: 47185bd68f3bbe58070094fd9d87bda3635476e4
2
+ SHA256:
3
+ metadata.gz: 1c252ffc4a609f7e29fffc86eb665c1032e2580b04277671157d44450becf3c0
4
+ data.tar.gz: a5c8ce1851371760195c3f51329fd3eb20f3e6d34e643d8ff833d18d77205b8c
5
5
  SHA512:
6
- metadata.gz: ef1608fd099a302e4977b3808d7adb39ea2718937b893545f147167579b9d0ff0dab8b231ab033650c340f0c66588f4fb4568704ea19af499979f0cb7c51276a
7
- data.tar.gz: e1a57cf92f6c81bb79bd7476d08eaecfab9c9d8c21f61f5d5b32e680cf70b0c9503c8bbc2ff7f2331f50a073c60010a55d105c37c0559cf66c82c439c47d7d5a
6
+ metadata.gz: e46064fe213ac097602e355f8d2e456ad16d070ada11ed7fc0e5f50c571f20514b2a97eb59cb9ce2c31d9a2d6f39cb05ef76edd5c65c2e27de60d2e39491decc
7
+ data.tar.gz: 468afcb058f36ab42e40f8267450156f5fd09b3f54ee5ee17e7f7b23ac976a5dd19322cb624b6d25bf4de606f37d1738d6cceeb927ebd42983519295af7f53f2
data/.travis.yml CHANGED
@@ -5,16 +5,16 @@ jdk:
5
5
 
6
6
  before_install:
7
7
  - openssl aes-256-cbc -K $encrypted_407f38382059_key -iv $encrypted_407f38382059_iv
8
- -in src/test/resources/secretkeys.tar.enc -out ./secretkeys.tar -d
9
- - tar xvf secretkeys.tar
8
+ -in src/test/resources/secrets.tar.enc -out ./secrets.tar -d
9
+ - tar xvf secrets.tar
10
10
 
11
11
  env:
12
12
  global:
13
13
  - GCP_EMAIL=unittest@embulk-input-gcs-test.iam.gserviceaccount.com
14
14
  - GCP_BUCKET=embulk-input-gcs-test
15
15
  - GCP_BUCKET_DIRECTORY=unittests_import
16
- - GCP_P12_KEYFILE=./embulk-input-gcs-test-9d51de6767a8.p12
17
- - GCP_JSON_KEYFILE=./embulk-input-gcs-test-65cfe930dca1.json
16
+ - GCP_P12_KEYFILE=./embulk-input-gcs-test-40bbfa15acd2.p12
17
+ - GCP_JSON_KEYFILE=./embulk-input-gcs-test-2492bb6e0f66.json
18
18
 
19
19
  script:
20
20
  - ./gradlew gem
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.3.1 - 2019-01-07
2
+ * [maintenance] Updated Google Cloud Storage API Client Library from v1-rev59-1.21.0 to 1.56.0 [#40](https://github.com/embulk/embulk-input-gcs/pull/40)
3
+ * [new feature] Refactor to use InputStream (to replace local file download)
4
+
1
5
  ## 0.3.0 - 2018-12-26
2
6
  * [maintenance] Use file name output feature in Embulk core to show file name in cmdout [#37](https://github.com/embulk/embulk-input-gcs/pull/37)
3
7
 
data/build.gradle CHANGED
@@ -1,11 +1,14 @@
1
1
  plugins {
2
- id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
2
+ id "com.jfrog.bintray" version "1.8.4"
3
+ id "com.github.jruby-gradle.base" version "1.6.0"
4
+ id "com.github.johnrengelman.shadow" version "4.0.3"
5
+ id "com.github.spotbugs" version "1.6.8"
4
6
  id "java"
5
7
  id "checkstyle"
6
8
  id "jacoco"
7
9
  }
8
10
  import com.github.jrubygradle.JRubyExec
11
+
9
12
  repositories {
10
13
  mavenCentral()
11
14
  jcenter()
@@ -17,38 +20,61 @@ configurations {
17
20
  sourceCompatibility = 1.8
18
21
  targetCompatibility = 1.8
19
22
 
20
- version = "0.3.0"
23
+ version = "0.3.1"
24
+
25
+ // Relocate Guava packages since it's incompatible with Guava's version from Embulk
26
+ shadowJar {
27
+ classifier = "shadow"
28
+ exclude "org/embulk/plugin/**"
29
+
30
+ dependencies {
31
+ include dependency("com.google.guava:guava")
32
+ include dependency("com.google.cloud:google-cloud-storage")
33
+ }
34
+
35
+ relocate "com.google.common", "embulk.gcs.com.google.common"
36
+ }
21
37
 
22
38
  dependencies {
23
39
  compile "org.embulk:embulk-core:0.9.12"
24
40
  provided "org.embulk:embulk-core:0.9.12"
25
41
 
26
- compile "com.google.http-client:google-http-client-jackson2:1.21.0"
27
- compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
42
+ compile "com.google.cloud:google-cloud-storage:1.56.0"
28
43
 
29
44
  testCompile "junit:junit:4.12"
30
45
  testCompile "org.embulk:embulk-core:0.9.12:tests"
31
46
  testCompile "org.embulk:embulk-standards:0.9.12"
47
+ testCompile "org.mockito:mockito-core:2.18.3"
32
48
  }
33
49
 
34
- task classpath(type: Copy, dependsOn: ["jar"]) {
50
+ task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
35
51
  doFirst { file("classpath").deleteDir() }
36
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
52
+ from (configurations.runtime - configurations.provided
53
+ + configurations.shadow
54
+ - files(shadowJar.getIncludedDependencies())
55
+ + files(shadowJar.archivePath))
37
56
  into "classpath"
38
57
  }
39
58
  clean { delete 'classpath' }
40
59
 
60
+ test {
61
+ jvmArgs '-XX:MaxPermSize=128M', '-Xmx2048m'
62
+ testLogging {
63
+ events "passed", "skipped", "failed"
64
+ }
65
+ }
66
+
41
67
  checkstyle {
42
68
  configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
43
69
  toolVersion = '6.14.1'
44
70
  }
45
71
  checkstyleMain {
46
72
  configFile = file("${project.rootDir}/config/checkstyle/default.xml")
47
- ignoreFailures = true
73
+ ignoreFailures = false
48
74
  }
49
75
  checkstyleTest {
50
76
  configFile = file("${project.rootDir}/config/checkstyle/default.xml")
51
- ignoreFailures = true
77
+ ignoreFailures = false
52
78
  }
53
79
  task checkstyle(type: Checkstyle) {
54
80
  classpath = sourceSets.main.output + sourceSets.test.output
@@ -56,12 +82,14 @@ task checkstyle(type: Checkstyle) {
56
82
  }
57
83
 
58
84
  task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
59
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
60
- script "build/gemspec"
85
+ script "gem"
86
+ scriptArgs "build", "build/gemspec"
61
87
  doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
62
88
  }
63
89
 
64
- task gemspec << { file("build/gemspec").write($/
90
+ task gemspec {
91
+ doLast {
92
+ file("build/gemspec").write($/
65
93
  Gem::Specification.new do |spec|
66
94
  spec.name = "${project.name}"
67
95
  spec.version = "${project.version}"
@@ -80,4 +108,5 @@ Gem::Specification.new do |spec|
80
108
  spec.add_development_dependency 'rake', ['>= 10.0']
81
109
  end
82
110
  /$)
111
+ }
83
112
  }
Binary file
@@ -1,6 +1,6 @@
1
- #Sun Jan 08 00:35:58 PST 2017
1
+ #Tue Dec 18 15:27:07 ICT 2018
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip
data/gradlew CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bash
1
+ #!/usr/bin/env sh
2
2
 
3
3
  ##############################################################################
4
4
  ##
@@ -6,12 +6,30 @@
6
6
  ##
7
7
  ##############################################################################
8
8
 
9
- # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
10
- DEFAULT_JVM_OPTS=""
9
+ # Attempt to set APP_HOME
10
+ # Resolve links: $0 may be a link
11
+ PRG="$0"
12
+ # Need this for relative symlinks.
13
+ while [ -h "$PRG" ] ; do
14
+ ls=`ls -ld "$PRG"`
15
+ link=`expr "$ls" : '.*-> \(.*\)$'`
16
+ if expr "$link" : '/.*' > /dev/null; then
17
+ PRG="$link"
18
+ else
19
+ PRG=`dirname "$PRG"`"/$link"
20
+ fi
21
+ done
22
+ SAVED="`pwd`"
23
+ cd "`dirname \"$PRG\"`/" >/dev/null
24
+ APP_HOME="`pwd -P`"
25
+ cd "$SAVED" >/dev/null
11
26
 
12
27
  APP_NAME="Gradle"
13
28
  APP_BASE_NAME=`basename "$0"`
14
29
 
30
+ # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
31
+ DEFAULT_JVM_OPTS=""
32
+
15
33
  # Use the maximum available, or set MAX_FD != -1 to use that value.
16
34
  MAX_FD="maximum"
17
35
 
@@ -30,6 +48,7 @@ die ( ) {
30
48
  cygwin=false
31
49
  msys=false
32
50
  darwin=false
51
+ nonstop=false
33
52
  case "`uname`" in
34
53
  CYGWIN* )
35
54
  cygwin=true
@@ -40,31 +59,11 @@ case "`uname`" in
40
59
  MINGW* )
41
60
  msys=true
42
61
  ;;
62
+ NONSTOP* )
63
+ nonstop=true
64
+ ;;
43
65
  esac
44
66
 
45
- # For Cygwin, ensure paths are in UNIX format before anything is touched.
46
- if $cygwin ; then
47
- [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
48
- fi
49
-
50
- # Attempt to set APP_HOME
51
- # Resolve links: $0 may be a link
52
- PRG="$0"
53
- # Need this for relative symlinks.
54
- while [ -h "$PRG" ] ; do
55
- ls=`ls -ld "$PRG"`
56
- link=`expr "$ls" : '.*-> \(.*\)$'`
57
- if expr "$link" : '/.*' > /dev/null; then
58
- PRG="$link"
59
- else
60
- PRG=`dirname "$PRG"`"/$link"
61
- fi
62
- done
63
- SAVED="`pwd`"
64
- cd "`dirname \"$PRG\"`/" >&-
65
- APP_HOME="`pwd -P`"
66
- cd "$SAVED" >&-
67
-
68
67
  CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
69
68
 
70
69
  # Determine the Java command to use to start the JVM.
@@ -90,7 +89,7 @@ location of your Java installation."
90
89
  fi
91
90
 
92
91
  # Increase the maximum file descriptors if we can.
93
- if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
92
+ if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
94
93
  MAX_FD_LIMIT=`ulimit -H -n`
95
94
  if [ $? -eq 0 ] ; then
96
95
  if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
@@ -114,6 +113,7 @@ fi
114
113
  if $cygwin ; then
115
114
  APP_HOME=`cygpath --path --mixed "$APP_HOME"`
116
115
  CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
116
+ JAVACMD=`cygpath --unix "$JAVACMD"`
117
117
 
118
118
  # We build the pattern for arguments to be converted via cygpath
119
119
  ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
@@ -154,11 +154,19 @@ if $cygwin ; then
154
154
  esac
155
155
  fi
156
156
 
157
- # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
158
- function splitJvmOpts() {
159
- JVM_OPTS=("$@")
157
+ # Escape application args
158
+ save ( ) {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
160
161
  }
161
- eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
162
- JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
162
+ APP_ARGS=$(save "$@")
163
+
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
163
171
 
164
- exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
172
+ exec "$JAVACMD" "$@"
data/gradlew.bat CHANGED
@@ -8,14 +8,14 @@
8
8
  @rem Set local scope for the variables with windows NT shell
9
9
  if "%OS%"=="Windows_NT" setlocal
10
10
 
11
- @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
12
- set DEFAULT_JVM_OPTS=
13
-
14
11
  set DIRNAME=%~dp0
15
12
  if "%DIRNAME%" == "" set DIRNAME=.
16
13
  set APP_BASE_NAME=%~n0
17
14
  set APP_HOME=%DIRNAME%
18
15
 
16
+ @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
17
+ set DEFAULT_JVM_OPTS=
18
+
19
19
  @rem Find java.exe
20
20
  if defined JAVA_HOME goto findJavaFromJavaHome
21
21
 
@@ -46,10 +46,9 @@ echo location of your Java installation.
46
46
  goto fail
47
47
 
48
48
  :init
49
- @rem Get command-line arguments, handling Windowz variants
49
+ @rem Get command-line arguments, handling Windows variants
50
50
 
51
51
  if not "%OS%" == "Windows_NT" goto win9xME_args
52
- if "%@eval[2+2]" == "4" goto 4NT_args
53
52
 
54
53
  :win9xME_args
55
54
  @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
60
59
  if "x%~1" == "x" goto execute
61
60
 
62
61
  set CMD_LINE_ARGS=%*
63
- goto execute
64
-
65
- :4NT_args
66
- @rem Get arguments from the 4NT Shell from JP Software
67
- set CMD_LINE_ARGS=%$
68
62
 
69
63
  :execute
70
64
  @rem Setup the command line
@@ -0,0 +1,108 @@
1
+ package org.embulk.input.gcs;
2
+
3
+ import com.google.api.client.util.SecurityUtils;
4
+ import com.google.api.services.storage.StorageScopes;
5
+ import com.google.auth.Credentials;
6
+ import com.google.auth.oauth2.ServiceAccountCredentials;
7
+ import com.google.cloud.storage.Storage;
8
+ import com.google.cloud.storage.StorageException;
9
+ import com.google.cloud.storage.StorageOptions;
10
+ import com.google.common.annotations.VisibleForTesting;
11
+ import com.google.common.collect.ImmutableList;
12
+ import org.embulk.config.Config;
13
+ import org.embulk.config.ConfigDefault;
14
+ import org.embulk.config.ConfigException;
15
+ import org.embulk.spi.unit.LocalFile;
16
+
17
+ import java.io.FileInputStream;
18
+ import java.io.IOException;
19
+ import java.io.InputStream;
20
+ import java.security.GeneralSecurityException;
21
+ import java.security.PrivateKey;
22
+ import java.util.Optional;
23
+
24
+ class AuthUtils
25
+ {
26
+ public enum AuthMethod
27
+ {
28
+ private_key, compute_engine, json_key
29
+ }
30
+
31
+ interface Task
32
+ {
33
+ @Config("auth_method")
34
+ @ConfigDefault("\"private_key\"")
35
+ AuthUtils.AuthMethod getAuthMethod();
36
+
37
+ @Config("service_account_email")
38
+ @ConfigDefault("null")
39
+ Optional<String> getServiceAccountEmail();
40
+
41
+ // kept for backward compatibility
42
+ @Config("p12_keyfile_fullpath")
43
+ @ConfigDefault("null")
44
+ Optional<String> getP12KeyfileFullpath();
45
+
46
+ @Config("p12_keyfile")
47
+ @ConfigDefault("null")
48
+ Optional<LocalFile> getP12Keyfile();
49
+ void setP12Keyfile(Optional<LocalFile> p12Keyfile);
50
+
51
+ @Config("json_keyfile")
52
+ @ConfigDefault("null")
53
+ Optional<LocalFile> getJsonKeyfile();
54
+ }
55
+
56
+ private AuthUtils()
57
+ {
58
+ }
59
+
60
+ static Storage newClient(final PluginTask task)
61
+ {
62
+ try {
63
+ final StorageOptions.Builder builder = StorageOptions.newBuilder();
64
+ switch (task.getAuthMethod()) {
65
+ case json_key:
66
+ builder.setCredentials(fromJson(task));
67
+ break;
68
+ case private_key:
69
+ builder.setCredentials(fromP12(task));
70
+ break;
71
+ default:
72
+ // compute_engine does not need credentials
73
+ break;
74
+ }
75
+ // test client to verify auth
76
+ final Storage client = builder.build().getService();
77
+ client.list(task.getBucket(), Storage.BlobListOption.pageSize(1));
78
+ return client;
79
+ }
80
+ catch (StorageException | IOException | GeneralSecurityException e) {
81
+ throw new ConfigException(e);
82
+ }
83
+ }
84
+
85
+ @VisibleForTesting
86
+ static Credentials fromP12(final Task task) throws IOException, GeneralSecurityException
87
+ {
88
+ final String path = task.getP12Keyfile().get().getPath().toString();
89
+ try (final InputStream p12InputStream = new FileInputStream(path)) {
90
+ final PrivateKey pk = SecurityUtils.loadPrivateKeyFromKeyStore(
91
+ SecurityUtils.getPkcs12KeyStore(), p12InputStream, "notasecret",
92
+ "privatekey", "notasecret");
93
+ return ServiceAccountCredentials.newBuilder()
94
+ .setClientEmail(task.getServiceAccountEmail().orElse(null))
95
+ .setPrivateKey(pk)
96
+ .setScopes(ImmutableList.of(StorageScopes.DEVSTORAGE_READ_ONLY))
97
+ .build();
98
+ }
99
+ }
100
+
101
+ @VisibleForTesting
102
+ static Credentials fromJson(final Task task) throws IOException
103
+ {
104
+ final String path = task.getJsonKeyfile().map(f -> f.getPath().toString()).get();
105
+ final InputStream jsonStream = new FileInputStream(path);
106
+ return ServiceAccountCredentials.fromStream(jsonStream);
107
+ }
108
+ }
@@ -3,6 +3,7 @@ package org.embulk.input.gcs;
3
3
  import com.fasterxml.jackson.annotation.JsonCreator;
4
4
  import com.fasterxml.jackson.annotation.JsonIgnore;
5
5
  import com.fasterxml.jackson.annotation.JsonProperty;
6
+ import com.google.common.base.Preconditions;
6
7
  import com.google.common.base.Throwables;
7
8
  import org.embulk.config.Config;
8
9
  import org.embulk.config.ConfigDefault;
@@ -122,7 +123,7 @@ public class FileList
122
123
  return this;
123
124
  }
124
125
 
125
- public Builder pathMatchPattern(String pattern)
126
+ public synchronized Builder pathMatchPattern(String pattern)
126
127
  {
127
128
  this.pathMatchPattern = Pattern.compile(pattern);
128
129
  return this;
@@ -168,7 +169,7 @@ public class FileList
168
169
  return true;
169
170
  }
170
171
 
171
- public FileList build()
172
+ public synchronized FileList build()
172
173
  {
173
174
  try {
174
175
  stream.close();
@@ -211,7 +212,7 @@ public class FileList
211
212
  @JsonProperty("tasks") List<List<Entry>> tasks,
212
213
  @JsonProperty("last") Optional<String> last)
213
214
  {
214
- this.data = data;
215
+ this.data = data.clone();
215
216
  this.tasks = tasks;
216
217
  this.last = last;
217
218
  }
@@ -241,7 +242,7 @@ public class FileList
241
242
  @Deprecated
242
243
  public byte[] getData()
243
244
  {
244
- return data;
245
+ return data.clone();
245
246
  }
246
247
 
247
248
  @JsonProperty("tasks")
@@ -258,7 +259,7 @@ public class FileList
258
259
  return last;
259
260
  }
260
261
 
261
- private class EntryList
262
+ private static class EntryList
262
263
  extends AbstractList<String>
263
264
  {
264
265
  private final byte[] data;
@@ -313,10 +314,12 @@ public class FileList
313
314
  private byte[] readNext()
314
315
  {
315
316
  try {
316
- stream.read(castBuffer.array());
317
- int n = castBuffer.getInt(0);
318
- byte[] b = new byte[n]; // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
319
- stream.read(b);
317
+ int n = stream.read(castBuffer.array());
318
+ Preconditions.checkArgument(n == castBuffer.capacity(), "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
319
+ int len = castBuffer.getInt(0);
320
+ byte[] b = new byte[len]; // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
321
+ n = stream.read(b);
322
+ Preconditions.checkArgument(n == len, "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
320
323
 
321
324
  current++;
322
325