embulk-input-gcs 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.travis.yml +4 -4
- data/CHANGELOG.md +4 -0
- data/build.gradle +41 -12
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +41 -33
- data/gradlew.bat +4 -10
- data/src/main/java/org/embulk/input/gcs/AuthUtils.java +108 -0
- data/src/main/java/org/embulk/input/gcs/FileList.java +12 -9
- data/src/main/java/org/embulk/input/gcs/GcsFileInput.java +46 -126
- data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java +3 -38
- data/src/main/java/org/embulk/input/gcs/PluginTask.java +1 -29
- data/src/main/java/org/embulk/input/gcs/RetryUtils.java +153 -0
- data/src/main/java/org/embulk/input/gcs/SingleFileProvider.java +30 -63
- data/src/test/java/org/embulk/input/gcs/TestAuthUtils.java +153 -0
- data/src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java +42 -94
- data/src/test/java/org/embulk/input/gcs/TestInputStreamReopener.java +145 -0
- data/src/test/java/org/embulk/input/gcs/TestRetryUtils.java +164 -0
- data/src/test/resources/secrets.tar.enc +0 -0
- metadata +53 -28
- data/secretkeys.tar +0 -0
- data/secretkeys.tar.enc +0 -0
- data/src/main/java/org/embulk/input/gcs/GcsAuthentication.java +0 -206
- data/src/test/java/org/embulk/input/gcs/TestGcsAuthentication.java +0 -185
- data/src/test/resources/secretkeys.tar.enc +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 1c252ffc4a609f7e29fffc86eb665c1032e2580b04277671157d44450becf3c0
|
4
|
+
data.tar.gz: a5c8ce1851371760195c3f51329fd3eb20f3e6d34e643d8ff833d18d77205b8c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e46064fe213ac097602e355f8d2e456ad16d070ada11ed7fc0e5f50c571f20514b2a97eb59cb9ce2c31d9a2d6f39cb05ef76edd5c65c2e27de60d2e39491decc
|
7
|
+
data.tar.gz: 468afcb058f36ab42e40f8267450156f5fd09b3f54ee5ee17e7f7b23ac976a5dd19322cb624b6d25bf4de606f37d1738d6cceeb927ebd42983519295af7f53f2
|
data/.travis.yml
CHANGED
@@ -5,16 +5,16 @@ jdk:
|
|
5
5
|
|
6
6
|
before_install:
|
7
7
|
- openssl aes-256-cbc -K $encrypted_407f38382059_key -iv $encrypted_407f38382059_iv
|
8
|
-
-in src/test/resources/
|
9
|
-
- tar xvf
|
8
|
+
-in src/test/resources/secrets.tar.enc -out ./secrets.tar -d
|
9
|
+
- tar xvf secrets.tar
|
10
10
|
|
11
11
|
env:
|
12
12
|
global:
|
13
13
|
- GCP_EMAIL=unittest@embulk-input-gcs-test.iam.gserviceaccount.com
|
14
14
|
- GCP_BUCKET=embulk-input-gcs-test
|
15
15
|
- GCP_BUCKET_DIRECTORY=unittests_import
|
16
|
-
- GCP_P12_KEYFILE=./embulk-input-gcs-test-
|
17
|
-
- GCP_JSON_KEYFILE=./embulk-input-gcs-test-
|
16
|
+
- GCP_P12_KEYFILE=./embulk-input-gcs-test-40bbfa15acd2.p12
|
17
|
+
- GCP_JSON_KEYFILE=./embulk-input-gcs-test-2492bb6e0f66.json
|
18
18
|
|
19
19
|
script:
|
20
20
|
- ./gradlew gem
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.3.1 - 2019-01-07
|
2
|
+
* [maintenance] Updated Google Cloud Storage API Client Library from v1-rev59-1.21.0 to 1.56.0 [#40](https://github.com/embulk/embulk-input-gcs/pull/40)
|
3
|
+
* [new feature] Refactor to use InputStream (to replace local file download)
|
4
|
+
|
1
5
|
## 0.3.0 - 2018-12-26
|
2
6
|
* [maintenance] Use file name output feature in Embulk core to show file name in cmdout [#37](https://github.com/embulk/embulk-input-gcs/pull/37)
|
3
7
|
|
data/build.gradle
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
plugins {
|
2
|
-
id "com.jfrog.bintray" version "1.
|
3
|
-
id "com.github.jruby-gradle.base" version "
|
2
|
+
id "com.jfrog.bintray" version "1.8.4"
|
3
|
+
id "com.github.jruby-gradle.base" version "1.6.0"
|
4
|
+
id "com.github.johnrengelman.shadow" version "4.0.3"
|
5
|
+
id "com.github.spotbugs" version "1.6.8"
|
4
6
|
id "java"
|
5
7
|
id "checkstyle"
|
6
8
|
id "jacoco"
|
7
9
|
}
|
8
10
|
import com.github.jrubygradle.JRubyExec
|
11
|
+
|
9
12
|
repositories {
|
10
13
|
mavenCentral()
|
11
14
|
jcenter()
|
@@ -17,38 +20,61 @@ configurations {
|
|
17
20
|
sourceCompatibility = 1.8
|
18
21
|
targetCompatibility = 1.8
|
19
22
|
|
20
|
-
version = "0.3.
|
23
|
+
version = "0.3.1"
|
24
|
+
|
25
|
+
// Relocate Guava packages since it's incompatible with Guava's version from Embulk
|
26
|
+
shadowJar {
|
27
|
+
classifier = "shadow"
|
28
|
+
exclude "org/embulk/plugin/**"
|
29
|
+
|
30
|
+
dependencies {
|
31
|
+
include dependency("com.google.guava:guava")
|
32
|
+
include dependency("com.google.cloud:google-cloud-storage")
|
33
|
+
}
|
34
|
+
|
35
|
+
relocate "com.google.common", "embulk.gcs.com.google.common"
|
36
|
+
}
|
21
37
|
|
22
38
|
dependencies {
|
23
39
|
compile "org.embulk:embulk-core:0.9.12"
|
24
40
|
provided "org.embulk:embulk-core:0.9.12"
|
25
41
|
|
26
|
-
compile "com.google.
|
27
|
-
compile ("com.google.apis:google-api-services-storage:v1-rev59-1.21.0") {exclude module: "guava-jdk5"}
|
42
|
+
compile "com.google.cloud:google-cloud-storage:1.56.0"
|
28
43
|
|
29
44
|
testCompile "junit:junit:4.12"
|
30
45
|
testCompile "org.embulk:embulk-core:0.9.12:tests"
|
31
46
|
testCompile "org.embulk:embulk-standards:0.9.12"
|
47
|
+
testCompile "org.mockito:mockito-core:2.18.3"
|
32
48
|
}
|
33
49
|
|
34
|
-
task classpath(type: Copy, dependsOn: ["jar"]) {
|
50
|
+
task classpath(type: Copy, dependsOn: ["jar", "shadowJar"]) {
|
35
51
|
doFirst { file("classpath").deleteDir() }
|
36
|
-
from (configurations.runtime - configurations.provided
|
52
|
+
from (configurations.runtime - configurations.provided
|
53
|
+
+ configurations.shadow
|
54
|
+
- files(shadowJar.getIncludedDependencies())
|
55
|
+
+ files(shadowJar.archivePath))
|
37
56
|
into "classpath"
|
38
57
|
}
|
39
58
|
clean { delete 'classpath' }
|
40
59
|
|
60
|
+
test {
|
61
|
+
jvmArgs '-XX:MaxPermSize=128M', '-Xmx2048m'
|
62
|
+
testLogging {
|
63
|
+
events "passed", "skipped", "failed"
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
41
67
|
checkstyle {
|
42
68
|
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
43
69
|
toolVersion = '6.14.1'
|
44
70
|
}
|
45
71
|
checkstyleMain {
|
46
72
|
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
47
|
-
ignoreFailures =
|
73
|
+
ignoreFailures = false
|
48
74
|
}
|
49
75
|
checkstyleTest {
|
50
76
|
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
51
|
-
ignoreFailures =
|
77
|
+
ignoreFailures = false
|
52
78
|
}
|
53
79
|
task checkstyle(type: Checkstyle) {
|
54
80
|
classpath = sourceSets.main.output + sourceSets.test.output
|
@@ -56,12 +82,14 @@ task checkstyle(type: Checkstyle) {
|
|
56
82
|
}
|
57
83
|
|
58
84
|
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
59
|
-
|
60
|
-
|
85
|
+
script "gem"
|
86
|
+
scriptArgs "build", "build/gemspec"
|
61
87
|
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
62
88
|
}
|
63
89
|
|
64
|
-
task gemspec
|
90
|
+
task gemspec {
|
91
|
+
doLast {
|
92
|
+
file("build/gemspec").write($/
|
65
93
|
Gem::Specification.new do |spec|
|
66
94
|
spec.name = "${project.name}"
|
67
95
|
spec.version = "${project.version}"
|
@@ -80,4 +108,5 @@ Gem::Specification.new do |spec|
|
|
80
108
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
81
109
|
end
|
82
110
|
/$)
|
111
|
+
}
|
83
112
|
}
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Tue Dec 18 15:27:07 ICT 2018
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-bin.zip
|
data/gradlew
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#!/usr/bin/env
|
1
|
+
#!/usr/bin/env sh
|
2
2
|
|
3
3
|
##############################################################################
|
4
4
|
##
|
@@ -6,12 +6,30 @@
|
|
6
6
|
##
|
7
7
|
##############################################################################
|
8
8
|
|
9
|
-
#
|
10
|
-
|
9
|
+
# Attempt to set APP_HOME
|
10
|
+
# Resolve links: $0 may be a link
|
11
|
+
PRG="$0"
|
12
|
+
# Need this for relative symlinks.
|
13
|
+
while [ -h "$PRG" ] ; do
|
14
|
+
ls=`ls -ld "$PRG"`
|
15
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
16
|
+
if expr "$link" : '/.*' > /dev/null; then
|
17
|
+
PRG="$link"
|
18
|
+
else
|
19
|
+
PRG=`dirname "$PRG"`"/$link"
|
20
|
+
fi
|
21
|
+
done
|
22
|
+
SAVED="`pwd`"
|
23
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
24
|
+
APP_HOME="`pwd -P`"
|
25
|
+
cd "$SAVED" >/dev/null
|
11
26
|
|
12
27
|
APP_NAME="Gradle"
|
13
28
|
APP_BASE_NAME=`basename "$0"`
|
14
29
|
|
30
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
31
|
+
DEFAULT_JVM_OPTS=""
|
32
|
+
|
15
33
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
34
|
MAX_FD="maximum"
|
17
35
|
|
@@ -30,6 +48,7 @@ die ( ) {
|
|
30
48
|
cygwin=false
|
31
49
|
msys=false
|
32
50
|
darwin=false
|
51
|
+
nonstop=false
|
33
52
|
case "`uname`" in
|
34
53
|
CYGWIN* )
|
35
54
|
cygwin=true
|
@@ -40,31 +59,11 @@ case "`uname`" in
|
|
40
59
|
MINGW* )
|
41
60
|
msys=true
|
42
61
|
;;
|
62
|
+
NONSTOP* )
|
63
|
+
nonstop=true
|
64
|
+
;;
|
43
65
|
esac
|
44
66
|
|
45
|
-
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
46
|
-
if $cygwin ; then
|
47
|
-
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
48
|
-
fi
|
49
|
-
|
50
|
-
# Attempt to set APP_HOME
|
51
|
-
# Resolve links: $0 may be a link
|
52
|
-
PRG="$0"
|
53
|
-
# Need this for relative symlinks.
|
54
|
-
while [ -h "$PRG" ] ; do
|
55
|
-
ls=`ls -ld "$PRG"`
|
56
|
-
link=`expr "$ls" : '.*-> \(.*\)$'`
|
57
|
-
if expr "$link" : '/.*' > /dev/null; then
|
58
|
-
PRG="$link"
|
59
|
-
else
|
60
|
-
PRG=`dirname "$PRG"`"/$link"
|
61
|
-
fi
|
62
|
-
done
|
63
|
-
SAVED="`pwd`"
|
64
|
-
cd "`dirname \"$PRG\"`/" >&-
|
65
|
-
APP_HOME="`pwd -P`"
|
66
|
-
cd "$SAVED" >&-
|
67
|
-
|
68
67
|
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
69
68
|
|
70
69
|
# Determine the Java command to use to start the JVM.
|
@@ -90,7 +89,7 @@ location of your Java installation."
|
|
90
89
|
fi
|
91
90
|
|
92
91
|
# Increase the maximum file descriptors if we can.
|
93
|
-
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
92
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
94
93
|
MAX_FD_LIMIT=`ulimit -H -n`
|
95
94
|
if [ $? -eq 0 ] ; then
|
96
95
|
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
@@ -114,6 +113,7 @@ fi
|
|
114
113
|
if $cygwin ; then
|
115
114
|
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
116
115
|
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
116
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
117
117
|
|
118
118
|
# We build the pattern for arguments to be converted via cygpath
|
119
119
|
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
@@ -154,11 +154,19 @@ if $cygwin ; then
|
|
154
154
|
esac
|
155
155
|
fi
|
156
156
|
|
157
|
-
#
|
158
|
-
|
159
|
-
|
157
|
+
# Escape application args
|
158
|
+
save ( ) {
|
159
|
+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
160
|
+
echo " "
|
160
161
|
}
|
161
|
-
|
162
|
-
|
162
|
+
APP_ARGS=$(save "$@")
|
163
|
+
|
164
|
+
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
165
|
+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
166
|
+
|
167
|
+
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
168
|
+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
169
|
+
cd "$(dirname "$0")"
|
170
|
+
fi
|
163
171
|
|
164
|
-
exec "$JAVACMD" "
|
172
|
+
exec "$JAVACMD" "$@"
|
data/gradlew.bat
CHANGED
@@ -8,14 +8,14 @@
|
|
8
8
|
@rem Set local scope for the variables with windows NT shell
|
9
9
|
if "%OS%"=="Windows_NT" setlocal
|
10
10
|
|
11
|
-
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
12
|
-
set DEFAULT_JVM_OPTS=
|
13
|
-
|
14
11
|
set DIRNAME=%~dp0
|
15
12
|
if "%DIRNAME%" == "" set DIRNAME=.
|
16
13
|
set APP_BASE_NAME=%~n0
|
17
14
|
set APP_HOME=%DIRNAME%
|
18
15
|
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS=
|
18
|
+
|
19
19
|
@rem Find java.exe
|
20
20
|
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
21
|
|
@@ -46,10 +46,9 @@ echo location of your Java installation.
|
|
46
46
|
goto fail
|
47
47
|
|
48
48
|
:init
|
49
|
-
@rem Get command-line arguments, handling
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
50
|
|
51
51
|
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
52
|
|
54
53
|
:win9xME_args
|
55
54
|
@rem Slurp the command line arguments.
|
@@ -60,11 +59,6 @@ set _SKIP=2
|
|
60
59
|
if "x%~1" == "x" goto execute
|
61
60
|
|
62
61
|
set CMD_LINE_ARGS=%*
|
63
|
-
goto execute
|
64
|
-
|
65
|
-
:4NT_args
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
-
set CMD_LINE_ARGS=%$
|
68
62
|
|
69
63
|
:execute
|
70
64
|
@rem Setup the command line
|
@@ -0,0 +1,108 @@
|
|
1
|
+
package org.embulk.input.gcs;
|
2
|
+
|
3
|
+
import com.google.api.client.util.SecurityUtils;
|
4
|
+
import com.google.api.services.storage.StorageScopes;
|
5
|
+
import com.google.auth.Credentials;
|
6
|
+
import com.google.auth.oauth2.ServiceAccountCredentials;
|
7
|
+
import com.google.cloud.storage.Storage;
|
8
|
+
import com.google.cloud.storage.StorageException;
|
9
|
+
import com.google.cloud.storage.StorageOptions;
|
10
|
+
import com.google.common.annotations.VisibleForTesting;
|
11
|
+
import com.google.common.collect.ImmutableList;
|
12
|
+
import org.embulk.config.Config;
|
13
|
+
import org.embulk.config.ConfigDefault;
|
14
|
+
import org.embulk.config.ConfigException;
|
15
|
+
import org.embulk.spi.unit.LocalFile;
|
16
|
+
|
17
|
+
import java.io.FileInputStream;
|
18
|
+
import java.io.IOException;
|
19
|
+
import java.io.InputStream;
|
20
|
+
import java.security.GeneralSecurityException;
|
21
|
+
import java.security.PrivateKey;
|
22
|
+
import java.util.Optional;
|
23
|
+
|
24
|
+
class AuthUtils
|
25
|
+
{
|
26
|
+
public enum AuthMethod
|
27
|
+
{
|
28
|
+
private_key, compute_engine, json_key
|
29
|
+
}
|
30
|
+
|
31
|
+
interface Task
|
32
|
+
{
|
33
|
+
@Config("auth_method")
|
34
|
+
@ConfigDefault("\"private_key\"")
|
35
|
+
AuthUtils.AuthMethod getAuthMethod();
|
36
|
+
|
37
|
+
@Config("service_account_email")
|
38
|
+
@ConfigDefault("null")
|
39
|
+
Optional<String> getServiceAccountEmail();
|
40
|
+
|
41
|
+
// kept for backward compatibility
|
42
|
+
@Config("p12_keyfile_fullpath")
|
43
|
+
@ConfigDefault("null")
|
44
|
+
Optional<String> getP12KeyfileFullpath();
|
45
|
+
|
46
|
+
@Config("p12_keyfile")
|
47
|
+
@ConfigDefault("null")
|
48
|
+
Optional<LocalFile> getP12Keyfile();
|
49
|
+
void setP12Keyfile(Optional<LocalFile> p12Keyfile);
|
50
|
+
|
51
|
+
@Config("json_keyfile")
|
52
|
+
@ConfigDefault("null")
|
53
|
+
Optional<LocalFile> getJsonKeyfile();
|
54
|
+
}
|
55
|
+
|
56
|
+
private AuthUtils()
|
57
|
+
{
|
58
|
+
}
|
59
|
+
|
60
|
+
static Storage newClient(final PluginTask task)
|
61
|
+
{
|
62
|
+
try {
|
63
|
+
final StorageOptions.Builder builder = StorageOptions.newBuilder();
|
64
|
+
switch (task.getAuthMethod()) {
|
65
|
+
case json_key:
|
66
|
+
builder.setCredentials(fromJson(task));
|
67
|
+
break;
|
68
|
+
case private_key:
|
69
|
+
builder.setCredentials(fromP12(task));
|
70
|
+
break;
|
71
|
+
default:
|
72
|
+
// compute_engine does not need credentials
|
73
|
+
break;
|
74
|
+
}
|
75
|
+
// test client to verify auth
|
76
|
+
final Storage client = builder.build().getService();
|
77
|
+
client.list(task.getBucket(), Storage.BlobListOption.pageSize(1));
|
78
|
+
return client;
|
79
|
+
}
|
80
|
+
catch (StorageException | IOException | GeneralSecurityException e) {
|
81
|
+
throw new ConfigException(e);
|
82
|
+
}
|
83
|
+
}
|
84
|
+
|
85
|
+
@VisibleForTesting
|
86
|
+
static Credentials fromP12(final Task task) throws IOException, GeneralSecurityException
|
87
|
+
{
|
88
|
+
final String path = task.getP12Keyfile().get().getPath().toString();
|
89
|
+
try (final InputStream p12InputStream = new FileInputStream(path)) {
|
90
|
+
final PrivateKey pk = SecurityUtils.loadPrivateKeyFromKeyStore(
|
91
|
+
SecurityUtils.getPkcs12KeyStore(), p12InputStream, "notasecret",
|
92
|
+
"privatekey", "notasecret");
|
93
|
+
return ServiceAccountCredentials.newBuilder()
|
94
|
+
.setClientEmail(task.getServiceAccountEmail().orElse(null))
|
95
|
+
.setPrivateKey(pk)
|
96
|
+
.setScopes(ImmutableList.of(StorageScopes.DEVSTORAGE_READ_ONLY))
|
97
|
+
.build();
|
98
|
+
}
|
99
|
+
}
|
100
|
+
|
101
|
+
@VisibleForTesting
|
102
|
+
static Credentials fromJson(final Task task) throws IOException
|
103
|
+
{
|
104
|
+
final String path = task.getJsonKeyfile().map(f -> f.getPath().toString()).get();
|
105
|
+
final InputStream jsonStream = new FileInputStream(path);
|
106
|
+
return ServiceAccountCredentials.fromStream(jsonStream);
|
107
|
+
}
|
108
|
+
}
|
@@ -3,6 +3,7 @@ package org.embulk.input.gcs;
|
|
3
3
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
4
4
|
import com.fasterxml.jackson.annotation.JsonIgnore;
|
5
5
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
6
|
+
import com.google.common.base.Preconditions;
|
6
7
|
import com.google.common.base.Throwables;
|
7
8
|
import org.embulk.config.Config;
|
8
9
|
import org.embulk.config.ConfigDefault;
|
@@ -122,7 +123,7 @@ public class FileList
|
|
122
123
|
return this;
|
123
124
|
}
|
124
125
|
|
125
|
-
public Builder pathMatchPattern(String pattern)
|
126
|
+
public synchronized Builder pathMatchPattern(String pattern)
|
126
127
|
{
|
127
128
|
this.pathMatchPattern = Pattern.compile(pattern);
|
128
129
|
return this;
|
@@ -168,7 +169,7 @@ public class FileList
|
|
168
169
|
return true;
|
169
170
|
}
|
170
171
|
|
171
|
-
public FileList build()
|
172
|
+
public synchronized FileList build()
|
172
173
|
{
|
173
174
|
try {
|
174
175
|
stream.close();
|
@@ -211,7 +212,7 @@ public class FileList
|
|
211
212
|
@JsonProperty("tasks") List<List<Entry>> tasks,
|
212
213
|
@JsonProperty("last") Optional<String> last)
|
213
214
|
{
|
214
|
-
this.data = data;
|
215
|
+
this.data = data.clone();
|
215
216
|
this.tasks = tasks;
|
216
217
|
this.last = last;
|
217
218
|
}
|
@@ -241,7 +242,7 @@ public class FileList
|
|
241
242
|
@Deprecated
|
242
243
|
public byte[] getData()
|
243
244
|
{
|
244
|
-
return data;
|
245
|
+
return data.clone();
|
245
246
|
}
|
246
247
|
|
247
248
|
@JsonProperty("tasks")
|
@@ -258,7 +259,7 @@ public class FileList
|
|
258
259
|
return last;
|
259
260
|
}
|
260
261
|
|
261
|
-
private class EntryList
|
262
|
+
private static class EntryList
|
262
263
|
extends AbstractList<String>
|
263
264
|
{
|
264
265
|
private final byte[] data;
|
@@ -313,10 +314,12 @@ public class FileList
|
|
313
314
|
private byte[] readNext()
|
314
315
|
{
|
315
316
|
try {
|
316
|
-
stream.read(castBuffer.array());
|
317
|
-
|
318
|
-
|
319
|
-
|
317
|
+
int n = stream.read(castBuffer.array());
|
318
|
+
Preconditions.checkArgument(n == castBuffer.capacity(), "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
|
319
|
+
int len = castBuffer.getInt(0);
|
320
|
+
byte[] b = new byte[len]; // here should be able to use a pooled buffer because read data is ignored if readNextString doesn't call this method
|
321
|
+
n = stream.read(b);
|
322
|
+
Preconditions.checkArgument(n == len, "Unexpected stream close, expecting %s bytes, but received %s bytes", castBuffer.capacity(), n);
|
320
323
|
|
321
324
|
current++;
|
322
325
|
|