embulk-output-gcs 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +23 -5
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/GcsAuthentication.java +184 -96
- data/src/main/java/org/embulk/output/GcsOutputPlugin.java +408 -278
- data/src/test/java/org/embulk/output/TestGcsAuthentication.java +161 -163
- data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java +399 -399
- metadata +5 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c5f66c9aebb242064e49a624cb08443142d7440b
|
|
4
|
+
data.tar.gz: bb3ef12c25a3bb768e2453012e351073566bf5b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 337c8a5c9cbf80008865a89dae91bad27de12915354b6937223ce72ef9f4295d089495ee577b2e29001486bb9cc1ff4217b23d1a7d7832bedaac9228ac3f224c
|
|
7
|
+
data.tar.gz: e26a507cebd8e93f87b0b5cd8b7c206901cc3ff9ca01b85a92e4ec721a2c84e164a009b134c259fb88a5e35ef8f02b47aaf2ebb9eab22d45948acb30cc2dadc4
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
id "jacoco"
|
|
6
7
|
}
|
|
7
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -16,18 +17,18 @@ configurations {
|
|
|
16
17
|
sourceCompatibility = 1.7
|
|
17
18
|
targetCompatibility = 1.7
|
|
18
19
|
|
|
19
|
-
version = "0.
|
|
20
|
+
version = "0.4.0"
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.6"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.6"
|
|
24
25
|
|
|
25
26
|
compile "com.google.http-client:google-http-client-jackson2:1.19.0"
|
|
26
27
|
compile ("com.google.apis:google-api-services-storage:v1-rev28-1.19.1") {exclude module: "guava-jdk5"}
|
|
27
28
|
|
|
28
29
|
testCompile "junit:junit:4.12"
|
|
29
|
-
testCompile "org.embulk:embulk-core:0.
|
|
30
|
-
testCompile "org.embulk:embulk-standards:0.
|
|
30
|
+
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
|
31
|
+
testCompile "org.embulk:embulk-standards:0.8.6"
|
|
31
32
|
}
|
|
32
33
|
|
|
33
34
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
@@ -37,6 +38,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
37
38
|
}
|
|
38
39
|
clean { delete 'classpath' }
|
|
39
40
|
|
|
41
|
+
checkstyle {
|
|
42
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
43
|
+
toolVersion = '6.14.1'
|
|
44
|
+
}
|
|
45
|
+
checkstyleMain {
|
|
46
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
47
|
+
ignoreFailures = true
|
|
48
|
+
}
|
|
49
|
+
checkstyleTest {
|
|
50
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
51
|
+
ignoreFailures = true
|
|
52
|
+
}
|
|
53
|
+
task checkstyle(type: Checkstyle) {
|
|
54
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
55
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
56
|
+
}
|
|
57
|
+
|
|
40
58
|
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
|
41
59
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
42
60
|
script "build/gemspec"
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="ClassTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
<module name="MethodTypeParameterName">
|
|
92
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
93
|
+
</module>
|
|
94
|
+
|
|
95
|
+
<module name="WhitespaceAround">
|
|
96
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
97
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
98
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
99
|
+
<property name="tokens" value="
|
|
100
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
101
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
102
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
103
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
104
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
105
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
106
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
107
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
108
|
+
</module>
|
|
109
|
+
</module>
|
|
110
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -1,110 +1,198 @@
|
|
|
1
1
|
package org.embulk.output;
|
|
2
2
|
|
|
3
|
-
import
|
|
4
|
-
import java.io.FileInputStream;
|
|
5
|
-
import java.io.IOException;
|
|
6
|
-
import java.util.Collections;
|
|
7
|
-
import com.google.api.client.http.apache.ApacheHttpTransport;
|
|
8
|
-
import com.google.api.services.storage.model.Objects;
|
|
9
|
-
import com.google.common.base.Optional;
|
|
10
|
-
import com.google.common.collect.ImmutableList;
|
|
11
|
-
import java.security.GeneralSecurityException;
|
|
3
|
+
import com.google.api.client.auth.oauth2.TokenResponseException;
|
|
12
4
|
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
|
13
5
|
import com.google.api.client.googleapis.compute.ComputeCredential;
|
|
6
|
+
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
|
7
|
+
import com.google.api.client.http.HttpRequestInitializer;
|
|
14
8
|
import com.google.api.client.http.HttpTransport;
|
|
9
|
+
import com.google.api.client.http.apache.ApacheHttpTransport;
|
|
15
10
|
import com.google.api.client.json.JsonFactory;
|
|
16
11
|
import com.google.api.client.json.jackson2.JacksonFactory;
|
|
17
|
-
import com.google.api.client.http.HttpRequestInitializer;
|
|
18
|
-
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
|
19
12
|
import com.google.api.services.storage.Storage;
|
|
20
13
|
import com.google.api.services.storage.StorageScopes;
|
|
14
|
+
import com.google.common.base.Optional;
|
|
15
|
+
import com.google.common.base.Throwables;
|
|
16
|
+
import com.google.common.collect.ImmutableList;
|
|
17
|
+
import org.embulk.config.ConfigException;
|
|
21
18
|
import org.embulk.spi.Exec;
|
|
19
|
+
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
|
20
|
+
import org.embulk.spi.util.RetryExecutor.Retryable;
|
|
22
21
|
import org.slf4j.Logger;
|
|
22
|
+
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
|
23
|
+
|
|
24
|
+
import java.io.File;
|
|
25
|
+
import java.io.FileInputStream;
|
|
26
|
+
|
|
27
|
+
import java.io.IOException;
|
|
28
|
+
import java.io.InterruptedIOException;
|
|
29
|
+
import java.security.GeneralSecurityException;
|
|
30
|
+
import java.util.Collections;
|
|
23
31
|
|
|
24
32
|
public class GcsAuthentication
|
|
25
33
|
{
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
34
|
+
private final Logger log = Exec.getLogger(GcsAuthentication.class);
|
|
35
|
+
private final Optional<String> serviceAccountEmail;
|
|
36
|
+
private final Optional<String> p12KeyFilePath;
|
|
37
|
+
private final Optional<String> jsonKeyFilePath;
|
|
38
|
+
private final String applicationName;
|
|
39
|
+
private final HttpTransport httpTransport;
|
|
40
|
+
private final JsonFactory jsonFactory;
|
|
41
|
+
private final HttpRequestInitializer credentials;
|
|
42
|
+
|
|
43
|
+
public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
|
44
|
+
Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
|
|
45
|
+
throws IOException, GeneralSecurityException
|
|
46
|
+
{
|
|
47
|
+
this.serviceAccountEmail = serviceAccountEmail;
|
|
48
|
+
this.p12KeyFilePath = p12KeyFilePath;
|
|
49
|
+
this.jsonKeyFilePath = jsonKeyFilePath;
|
|
50
|
+
this.applicationName = applicationName;
|
|
51
|
+
|
|
52
|
+
this.httpTransport = new ApacheHttpTransport.Builder().build();
|
|
53
|
+
this.jsonFactory = new JacksonFactory();
|
|
54
|
+
|
|
55
|
+
if (authMethod.equals("compute_engine")) {
|
|
56
|
+
this.credentials = getComputeCredential();
|
|
57
|
+
}
|
|
58
|
+
else if (authMethod.toLowerCase().equals("json_key")) {
|
|
59
|
+
this.credentials = getServiceAccountCredentialFromJsonFile();
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
this.credentials = getServiceAccountCredential();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* @see https://developers.google.com/accounts/docs/OAuth2ServiceAccount#authorizingrequests
|
|
68
|
+
*/
|
|
69
|
+
private GoogleCredential getServiceAccountCredential() throws IOException, GeneralSecurityException
|
|
70
|
+
{
|
|
71
|
+
// @see https://cloud.google.com/compute/docs/api/how-tos/authorization
|
|
72
|
+
// @see https://developers.google.com/resources/api-libraries/documentation/storage/v1/java/latest/com/google/api/services/storage/STORAGE_SCOPE.html
|
|
73
|
+
// @see https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/BigqueryScopes.html
|
|
74
|
+
return new GoogleCredential.Builder()
|
|
75
|
+
.setTransport(httpTransport)
|
|
76
|
+
.setJsonFactory(jsonFactory)
|
|
77
|
+
.setServiceAccountId(serviceAccountEmail.orNull())
|
|
78
|
+
.setServiceAccountScopes(
|
|
79
|
+
ImmutableList.of(
|
|
80
|
+
StorageScopes.DEVSTORAGE_READ_WRITE
|
|
81
|
+
)
|
|
82
|
+
)
|
|
83
|
+
.setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.get()))
|
|
84
|
+
.build();
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
|
|
88
|
+
{
|
|
89
|
+
FileInputStream stream = new FileInputStream(jsonKeyFilePath.get());
|
|
90
|
+
|
|
91
|
+
return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
|
|
92
|
+
.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE));
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
|
|
97
|
+
* @see https://developers.google.com/accounts/docs/OAuth2
|
|
98
|
+
*/
|
|
99
|
+
private ComputeCredential getComputeCredential() throws IOException
|
|
100
|
+
{
|
|
101
|
+
ComputeCredential credential = new ComputeCredential.Builder(httpTransport, jsonFactory)
|
|
102
|
+
.build();
|
|
103
|
+
credential.refreshToken();
|
|
104
|
+
|
|
105
|
+
return credential;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
public Storage getGcsClient(final String bucket, int maxConnectionRetry) throws ConfigException, IOException
|
|
109
|
+
{
|
|
110
|
+
try {
|
|
111
|
+
return retryExecutor()
|
|
112
|
+
.withRetryLimit(maxConnectionRetry)
|
|
113
|
+
.withInitialRetryWait(500)
|
|
114
|
+
.withMaxRetryWait(30 * 1000)
|
|
115
|
+
.runInterruptible(new Retryable<Storage>() {
|
|
116
|
+
@Override
|
|
117
|
+
public Storage call() throws IOException, RetryGiveupException
|
|
118
|
+
{
|
|
119
|
+
Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
|
|
120
|
+
.setApplicationName(applicationName)
|
|
121
|
+
.build();
|
|
122
|
+
|
|
123
|
+
// For throw ConfigException when authentication is fail.
|
|
124
|
+
long maxResults = 1;
|
|
125
|
+
client.objects().list(bucket).setMaxResults(maxResults).execute();
|
|
126
|
+
|
|
127
|
+
return client;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
@Override
|
|
131
|
+
public boolean isRetryableException(Exception exception)
|
|
132
|
+
{
|
|
133
|
+
if (exception instanceof GoogleJsonResponseException || exception instanceof TokenResponseException) {
|
|
134
|
+
int statusCode;
|
|
135
|
+
if (exception instanceof GoogleJsonResponseException) {
|
|
136
|
+
if (((GoogleJsonResponseException) exception).getDetails() == null) {
|
|
137
|
+
String content = "";
|
|
138
|
+
if (((GoogleJsonResponseException) exception).getContent() != null) {
|
|
139
|
+
content = ((GoogleJsonResponseException) exception).getContent();
|
|
140
|
+
}
|
|
141
|
+
log.warn("Invalid response was returned : {}", content);
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
statusCode = ((GoogleJsonResponseException) exception).getDetails().getCode();
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
statusCode = ((TokenResponseException) exception).getStatusCode();
|
|
148
|
+
}
|
|
149
|
+
if (statusCode / 100 == 4) {
|
|
150
|
+
return false;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return true;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
@Override
|
|
157
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
|
158
|
+
throws RetryGiveupException
|
|
159
|
+
{
|
|
160
|
+
String message = String.format("GCS GET request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
|
|
161
|
+
retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
|
|
162
|
+
if (retryCount % 3 == 0) {
|
|
163
|
+
log.warn(message, exception);
|
|
164
|
+
}
|
|
165
|
+
else {
|
|
166
|
+
log.warn(message);
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
@Override
|
|
171
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
|
172
|
+
throws RetryGiveupException
|
|
173
|
+
{
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
catch (RetryGiveupException ex) {
|
|
178
|
+
if (ex.getCause() instanceof GoogleJsonResponseException || ex.getCause() instanceof TokenResponseException) {
|
|
179
|
+
int statusCode = 0;
|
|
180
|
+
if (ex.getCause() instanceof GoogleJsonResponseException) {
|
|
181
|
+
if (((GoogleJsonResponseException) ex.getCause()).getDetails() != null) {
|
|
182
|
+
statusCode = ((GoogleJsonResponseException) ex.getCause()).getDetails().getCode();
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
else if (ex.getCause() instanceof TokenResponseException) {
|
|
186
|
+
statusCode = ((TokenResponseException) ex.getCause()).getStatusCode();
|
|
187
|
+
}
|
|
188
|
+
if (statusCode / 100 == 4) {
|
|
189
|
+
throw new ConfigException(ex);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
throw Throwables.propagate(ex);
|
|
193
|
+
}
|
|
194
|
+
catch (InterruptedException ex) {
|
|
195
|
+
throw new InterruptedIOException();
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
}
|