embulk-output-gcs 0.3.0 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +23 -5
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/GcsAuthentication.java +184 -96
- data/src/main/java/org/embulk/output/GcsOutputPlugin.java +408 -278
- data/src/test/java/org/embulk/output/TestGcsAuthentication.java +161 -163
- data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java +399 -399
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c5f66c9aebb242064e49a624cb08443142d7440b
|
|
4
|
+
data.tar.gz: bb3ef12c25a3bb768e2453012e351073566bf5b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 337c8a5c9cbf80008865a89dae91bad27de12915354b6937223ce72ef9f4295d089495ee577b2e29001486bb9cc1ff4217b23d1a7d7832bedaac9228ac3f224c
|
|
7
|
+
data.tar.gz: e26a507cebd8e93f87b0b5cd8b7c206901cc3ff9ca01b85a92e4ec721a2c84e164a009b134c259fb88a5e35ef8f02b47aaf2ebb9eab22d45948acb30cc2dadc4
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
id "jacoco"
|
|
6
7
|
}
|
|
7
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -16,18 +17,18 @@ configurations {
|
|
|
16
17
|
sourceCompatibility = 1.7
|
|
17
18
|
targetCompatibility = 1.7
|
|
18
19
|
|
|
19
|
-
version = "0.
|
|
20
|
+
version = "0.4.0"
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.6"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.6"
|
|
24
25
|
|
|
25
26
|
compile "com.google.http-client:google-http-client-jackson2:1.19.0"
|
|
26
27
|
compile ("com.google.apis:google-api-services-storage:v1-rev28-1.19.1") {exclude module: "guava-jdk5"}
|
|
27
28
|
|
|
28
29
|
testCompile "junit:junit:4.12"
|
|
29
|
-
testCompile "org.embulk:embulk-core:0.
|
|
30
|
-
testCompile "org.embulk:embulk-standards:0.
|
|
30
|
+
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
|
31
|
+
testCompile "org.embulk:embulk-standards:0.8.6"
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
@@ -37,6 +38,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
37
38
|
}
|
|
38
39
|
clean { delete 'classpath' }
|
|
39
40
|
|
|
41
|
+
checkstyle {
|
|
42
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
43
|
+
toolVersion = '6.14.1'
|
|
44
|
+
}
|
|
45
|
+
checkstyleMain {
|
|
46
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
47
|
+
ignoreFailures = true
|
|
48
|
+
}
|
|
49
|
+
checkstyleTest {
|
|
50
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
51
|
+
ignoreFailures = true
|
|
52
|
+
}
|
|
53
|
+
task checkstyle(type: Checkstyle) {
|
|
54
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
55
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
56
|
+
}
|
|
57
|
+
|
|
40
58
|
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
|
41
59
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
42
60
|
script "build/gemspec"
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="ClassTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
<module name="MethodTypeParameterName">
|
|
92
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
93
|
+
</module>
|
|
94
|
+
|
|
95
|
+
<module name="WhitespaceAround">
|
|
96
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
97
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
98
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
99
|
+
<property name="tokens" value="
|
|
100
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
101
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
102
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
103
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
104
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
105
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
106
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
107
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
108
|
+
</module>
|
|
109
|
+
</module>
|
|
110
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -1,110 +1,198 @@
|
|
|
1
1
|
package org.embulk.output;
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import java.io.FileInputStream;
|
|
5
|
-
import java.io.IOException;
|
|
6
|
-
import java.util.Collections;
|
|
7
|
-
import com.google.api.client.http.apache.ApacheHttpTransport;
|
|
8
|
-
import com.google.api.services.storage.model.Objects;
|
|
9
|
-
import com.google.common.base.Optional;
|
|
10
|
-
import com.google.common.collect.ImmutableList;
|
|
11
|
-
import java.security.GeneralSecurityException;
|
|
3
|
+
import com.google.api.client.auth.oauth2.TokenResponseException;
|
|
12
4
|
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
|
13
5
|
import com.google.api.client.googleapis.compute.ComputeCredential;
|
|
6
|
+
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
|
7
|
+
import com.google.api.client.http.HttpRequestInitializer;
|
|
14
8
|
import com.google.api.client.http.HttpTransport;
|
|
9
|
+
import com.google.api.client.http.apache.ApacheHttpTransport;
|
|
15
10
|
import com.google.api.client.json.JsonFactory;
|
|
16
11
|
import com.google.api.client.json.jackson2.JacksonFactory;
|
|
17
|
-
import com.google.api.client.http.HttpRequestInitializer;
|
|
18
|
-
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
|
19
12
|
import com.google.api.services.storage.Storage;
|
|
20
13
|
import com.google.api.services.storage.StorageScopes;
|
|
14
|
+
import com.google.common.base.Optional;
|
|
15
|
+
import com.google.common.base.Throwables;
|
|
16
|
+
import com.google.common.collect.ImmutableList;
|
|
17
|
+
import org.embulk.config.ConfigException;
|
|
21
18
|
import org.embulk.spi.Exec;
|
|
19
|
+
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
|
20
|
+
import org.embulk.spi.util.RetryExecutor.Retryable;
|
|
22
21
|
import org.slf4j.Logger;
|
|
22
|
+
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
|
23
|
+
|
|
24
|
+
import java.io.File;
|
|
25
|
+
import java.io.FileInputStream;
|
|
26
|
+
|
|
27
|
+
import java.io.IOException;
|
|
28
|
+
import java.io.InterruptedIOException;
|
|
29
|
+
import java.security.GeneralSecurityException;
|
|
30
|
+
import java.util.Collections;
|
|
23
31
|
|
|
24
32
|
public class GcsAuthentication
|
|
25
33
|
{
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
34
|
+
private final Logger log = Exec.getLogger(GcsAuthentication.class);
|
|
35
|
+
private final Optional<String> serviceAccountEmail;
|
|
36
|
+
private final Optional<String> p12KeyFilePath;
|
|
37
|
+
private final Optional<String> jsonKeyFilePath;
|
|
38
|
+
private final String applicationName;
|
|
39
|
+
private final HttpTransport httpTransport;
|
|
40
|
+
private final JsonFactory jsonFactory;
|
|
41
|
+
private final HttpRequestInitializer credentials;
|
|
42
|
+
|
|
43
|
+
public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
|
44
|
+
Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
|
|
45
|
+
throws IOException, GeneralSecurityException
|
|
46
|
+
{
|
|
47
|
+
this.serviceAccountEmail = serviceAccountEmail;
|
|
48
|
+
this.p12KeyFilePath = p12KeyFilePath;
|
|
49
|
+
this.jsonKeyFilePath = jsonKeyFilePath;
|
|
50
|
+
this.applicationName = applicationName;
|
|
51
|
+
|
|
52
|
+
this.httpTransport = new ApacheHttpTransport.Builder().build();
|
|
53
|
+
this.jsonFactory = new JacksonFactory();
|
|
54
|
+
|
|
55
|
+
if (authMethod.equals("compute_engine")) {
|
|
56
|
+
this.credentials = getComputeCredential();
|
|
57
|
+
}
|
|
58
|
+
else if (authMethod.toLowerCase().equals("json_key")) {
|
|
59
|
+
this.credentials = getServiceAccountCredentialFromJsonFile();
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
this.credentials = getServiceAccountCredential();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @see https://developers.google.com/accounts/docs/OAuth2ServiceAccount#authorizingrequests
|
|
68
|
+
*/
|
|
69
|
+
private GoogleCredential getServiceAccountCredential() throws IOException, GeneralSecurityException
|
|
70
|
+
{
|
|
71
|
+
// @see https://cloud.google.com/compute/docs/api/how-tos/authorization
|
|
72
|
+
// @see https://developers.google.com/resources/api-libraries/documentation/storage/v1/java/latest/com/google/api/services/storage/STORAGE_SCOPE.html
|
|
73
|
+
// @see https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/BigqueryScopes.html
|
|
74
|
+
return new GoogleCredential.Builder()
|
|
75
|
+
.setTransport(httpTransport)
|
|
76
|
+
.setJsonFactory(jsonFactory)
|
|
77
|
+
.setServiceAccountId(serviceAccountEmail.orNull())
|
|
78
|
+
.setServiceAccountScopes(
|
|
79
|
+
ImmutableList.of(
|
|
80
|
+
StorageScopes.DEVSTORAGE_READ_WRITE
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
.setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.get()))
|
|
84
|
+
.build();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
|
|
88
|
+
{
|
|
89
|
+
FileInputStream stream = new FileInputStream(jsonKeyFilePath.get());
|
|
90
|
+
|
|
91
|
+
return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
|
|
92
|
+
.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
|
|
97
|
+
* @see https://developers.google.com/accounts/docs/OAuth2
|
|
98
|
+
*/
|
|
99
|
+
private ComputeCredential getComputeCredential() throws IOException
|
|
100
|
+
{
|
|
101
|
+
ComputeCredential credential = new ComputeCredential.Builder(httpTransport, jsonFactory)
|
|
102
|
+
.build();
|
|
103
|
+
credential.refreshToken();
|
|
104
|
+
|
|
105
|
+
return credential;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
public Storage getGcsClient(final String bucket, int maxConnectionRetry) throws ConfigException, IOException
|
|
109
|
+
{
|
|
110
|
+
try {
|
|
111
|
+
return retryExecutor()
|
|
112
|
+
.withRetryLimit(maxConnectionRetry)
|
|
113
|
+
.withInitialRetryWait(500)
|
|
114
|
+
.withMaxRetryWait(30 * 1000)
|
|
115
|
+
.runInterruptible(new Retryable<Storage>() {
|
|
116
|
+
@Override
|
|
117
|
+
public Storage call() throws IOException, RetryGiveupException
|
|
118
|
+
{
|
|
119
|
+
Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
|
|
120
|
+
.setApplicationName(applicationName)
|
|
121
|
+
.build();
|
|
122
|
+
|
|
123
|
+
// For throw ConfigException when authentication is fail.
|
|
124
|
+
long maxResults = 1;
|
|
125
|
+
client.objects().list(bucket).setMaxResults(maxResults).execute();
|
|
126
|
+
|
|
127
|
+
return client;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
@Override
|
|
131
|
+
public boolean isRetryableException(Exception exception)
|
|
132
|
+
{
|
|
133
|
+
if (exception instanceof GoogleJsonResponseException || exception instanceof TokenResponseException) {
|
|
134
|
+
int statusCode;
|
|
135
|
+
if (exception instanceof GoogleJsonResponseException) {
|
|
136
|
+
if (((GoogleJsonResponseException) exception).getDetails() == null) {
|
|
137
|
+
String content = "";
|
|
138
|
+
if (((GoogleJsonResponseException) exception).getContent() != null) {
|
|
139
|
+
content = ((GoogleJsonResponseException) exception).getContent();
|
|
140
|
+
}
|
|
141
|
+
log.warn("Invalid response was returned : {}", content);
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
statusCode = ((GoogleJsonResponseException) exception).getDetails().getCode();
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
statusCode = ((TokenResponseException) exception).getStatusCode();
|
|
148
|
+
}
|
|
149
|
+
if (statusCode / 100 == 4) {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return true;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
@Override
|
|
157
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
|
158
|
+
throws RetryGiveupException
|
|
159
|
+
{
|
|
160
|
+
String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
|
|
161
|
+
retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
|
|
162
|
+
if (retryCount % 3 == 0) {
|
|
163
|
+
log.warn(message, exception);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
log.warn(message);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
@Override
|
|
171
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
|
172
|
+
throws RetryGiveupException
|
|
173
|
+
{
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (RetryGiveupException ex) {
|
|
178
|
+
if (ex.getCause() instanceof GoogleJsonResponseException || ex.getCause() instanceof TokenResponseException) {
|
|
179
|
+
int statusCode = 0;
|
|
180
|
+
if (ex.getCause() instanceof GoogleJsonResponseException) {
|
|
181
|
+
if (((GoogleJsonResponseException) ex.getCause()).getDetails() != null) {
|
|
182
|
+
statusCode = ((GoogleJsonResponseException) ex.getCause()).getDetails().getCode();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
else if (ex.getCause() instanceof TokenResponseException) {
|
|
186
|
+
statusCode = ((TokenResponseException) ex.getCause()).getStatusCode();
|
|
187
|
+
}
|
|
188
|
+
if (statusCode / 100 == 4) {
|
|
189
|
+
throw new ConfigException(ex);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
throw Throwables.propagate(ex);
|
|
193
|
+
}
|
|
194
|
+
catch (InterruptedException ex) {
|
|
195
|
+
throw new InterruptedIOException();
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|