embulk-output-bigquery 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +42 -0
- data/README.md +48 -7
- data/build.gradle +1 -1
- data/classpath/embulk-output-bigquery-0.1.10.jar +0 -0
- data/src/main/java/org/embulk/output/BigqueryAuthentication.java +17 -3
- data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java +17 -1
- data/src/main/java/org/embulk/output/BigqueryWriter.java +8 -2
- metadata +4 -3
- data/classpath/embulk-output-bigquery-0.1.9.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a87dead2fa7962e1c2b07c86adc52939c2ed4205
|
4
|
+
data.tar.gz: 34db5af6375ce61615b5aeafb94cf9eb11e576c0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 46ce7c4c2a5b5e1213724bf571bdf3764320d34cc787f79bc6ef20cef4681f8a402c600bdef807311f550045ce0c95f5a6eb5abc4262fce746705f86119ccc76
|
7
|
+
data.tar.gz: bc5bfb7050882ef5312a78ee6c0d746e76bae0045baa0c82cd9f4e2c67754aeccc834ce1b7864ad8e105d7e56f2d3eec63131c7c0feee7a79dd0f18b24293611
|
data/ChangeLog
ADDED
@@ -0,0 +1,42 @@
|
|
1
|
+
Release 0.1.10 - 2015-10-06
|
2
|
+
|
3
|
+
* Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account
|
4
|
+
|
5
|
+
Release 0.1.9 - 2015-08-19
|
6
|
+
|
7
|
+
* Upgraded embulk version to 0.7.1
|
8
|
+
|
9
|
+
Release 0.1.8 - 2015-08-19
|
10
|
+
|
11
|
+
* Supported mapreduce-executor. @frsyuki thanks!
|
12
|
+
* Fixed job_id generation logic
|
13
|
+
* Refactored
|
14
|
+
|
15
|
+
Release 0.1.7 - 2015-03-20
|
16
|
+
|
17
|
+
* Added allow_quoted_newlines option
|
18
|
+
* Upgraded embulk version to 0.6.8
|
19
|
+
|
20
|
+
Release 0.1.6 - 2015-04-23
|
21
|
+
|
22
|
+
* Added ignore_unknown_values option to job_id generation logic.
|
23
|
+
|
24
|
+
Release 0.1.5 - 2015-04-23
|
25
|
+
|
26
|
+
* Added ignore_unknown_values option. @takus thanks!
|
27
|
+
|
28
|
+
Release 0.1.4 - 2015-04-21
|
29
|
+
|
30
|
+
* Added prevent_duplicate_insert option
|
31
|
+
|
32
|
+
Release 0.1.3 - 2015-04-06
|
33
|
+
|
34
|
+
* Added new auth method - pre-defined access token of GCE(Google Compute Engine)
|
35
|
+
* Updated Google provided libraries
|
36
|
+
* http-client:google-http-client-jackson2 from 1.19.0 to 1.20.0
|
37
|
+
* apis:google-api-services-bigquery from v2-rev193-1.19.1 to v2-rev205-1.20.0
|
38
|
+
|
39
|
+
Release 0.1.2 - 2015-04-01
|
40
|
+
|
41
|
+
* Changed bulk-load method from "via GCS" to direct-insert
|
42
|
+
* added dynamic table creationg option
|
data/README.md
CHANGED
@@ -28,9 +28,10 @@ OAuth flow for installed applications.
|
|
28
28
|
|
29
29
|
| name | type | required? | default | description |
|
30
30
|
|:--------------------------|:------------|:-----------|:-------------|:-----------------------|
|
31
|
-
| auth_method | string | optional | "private_key" | `private_key` or `compute_engine`
|
31
|
+
| auth_method | string | optional | "private_key" | `private_key` , `json_key` or `compute_engine`
|
32
32
|
| service_account_email | string | required when auth_method is private_key | | Your Google service account email
|
33
33
|
| p12_keyfile | string | required when auth_method is private_key | | Fullpath of private key in P12(PKCS12) format |
|
34
|
+
| json_keyfile | string | required when auth_method is json_key | | Fullpath of json key |
|
34
35
|
| sequence_format | string | optional | %03d.%02d | |
|
35
36
|
| file_ext | string | optional | | e.g. ".csv.gz" ".json.gz" |
|
36
37
|
| project | string | required | | project_id |
|
@@ -81,18 +82,58 @@ out:
|
|
81
82
|
|
82
83
|
### Authentication
|
83
84
|
|
84
|
-
There are
|
85
|
+
There are three methods supported to fetch access token for the service account.
|
85
86
|
|
86
|
-
1. Public-Private key pair
|
87
|
-
2.
|
87
|
+
1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
|
88
|
+
2. JSON key of GCP(Google Cloud Platform)'s service account
|
89
|
+
3. Pre-defined access token (Google Compute Engine only)
|
88
90
|
|
89
|
-
|
91
|
+
#### Public-Private key pair of GCP's service account
|
92
|
+
|
93
|
+
You first need to create a service account (client ID),
|
90
94
|
download its private key and deploy the key with embulk.
|
91
95
|
|
96
|
+
```yaml
|
97
|
+
out:
|
98
|
+
type: bigquery
|
99
|
+
auth_method: private_key # default
|
100
|
+
service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
|
101
|
+
p12_keyfile: /path/to/p12_keyfile.p12
|
102
|
+
```
|
103
|
+
|
104
|
+
#### JSON key of GCP's service account
|
105
|
+
|
106
|
+
You first need to create a service account (client ID),
|
107
|
+
download its json key and deploy the key with embulk.
|
108
|
+
|
109
|
+
```yaml
|
110
|
+
out:
|
111
|
+
type: bigquery
|
112
|
+
auth_method: json_key
|
113
|
+
json_keyfile: /path/to/json_keyfile.json
|
114
|
+
```
|
115
|
+
|
116
|
+
You can also embed contents of json_keyfile at config.yml.
|
117
|
+
|
118
|
+
```yaml
|
119
|
+
in:
|
120
|
+
type: gcs
|
121
|
+
auth_method: json_key
|
122
|
+
json_keyfile:
|
123
|
+
content: |
|
124
|
+
{
|
125
|
+
"private_key_id": "123456789",
|
126
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
|
127
|
+
"client_email": "..."
|
128
|
+
}
|
129
|
+
```
|
130
|
+
|
131
|
+
#### Pre-defined access token(GCE only)
|
132
|
+
|
92
133
|
On the other hand, you don't need to explicitly create a service account for embulk when you
|
93
|
-
run embulk in Google Compute Engine.
|
134
|
+
run embulk in Google Compute Engine. In this third authentication method, you need to
|
94
135
|
add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
|
95
|
-
Compute Engine instance, then you can configure embulk like this.
|
136
|
+
Compute Engine VM instance, then you can configure embulk like this.
|
96
137
|
|
97
138
|
```yaml
|
98
139
|
out:
|
data/build.gradle
CHANGED
Binary file
|
@@ -1,12 +1,13 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
3
|
import java.io.File;
|
4
|
+
import java.io.FileInputStream;
|
4
5
|
import java.io.IOException;
|
6
|
+
import java.util.Collections;
|
7
|
+
import java.security.GeneralSecurityException;
|
5
8
|
|
6
9
|
import com.google.common.base.Optional;
|
7
10
|
import com.google.common.collect.ImmutableList;
|
8
|
-
import java.security.GeneralSecurityException;
|
9
|
-
|
10
11
|
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
11
12
|
import com.google.api.client.googleapis.compute.ComputeCredential;
|
12
13
|
import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
|
@@ -26,16 +27,19 @@ public class BigqueryAuthentication
|
|
26
27
|
private final Logger log = Exec.getLogger(BigqueryAuthentication.class);
|
27
28
|
private final Optional<String> serviceAccountEmail;
|
28
29
|
private final Optional<String> p12KeyFilePath;
|
30
|
+
private final Optional<String> jsonKeyFilePath;
|
29
31
|
private final String applicationName;
|
30
32
|
private final HttpTransport httpTransport;
|
31
33
|
private final JsonFactory jsonFactory;
|
32
34
|
private final HttpRequestInitializer credentials;
|
33
35
|
|
34
|
-
public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
36
|
+
public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
37
|
+
Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
|
35
38
|
throws IOException, GeneralSecurityException
|
36
39
|
{
|
37
40
|
this.serviceAccountEmail = serviceAccountEmail;
|
38
41
|
this.p12KeyFilePath = p12KeyFilePath;
|
42
|
+
this.jsonKeyFilePath = jsonKeyFilePath;
|
39
43
|
this.applicationName = applicationName;
|
40
44
|
|
41
45
|
this.httpTransport = GoogleNetHttpTransport.newTrustedTransport();
|
@@ -43,6 +47,8 @@ public class BigqueryAuthentication
|
|
43
47
|
|
44
48
|
if (authMethod.toLowerCase().equals("compute_engine")) {
|
45
49
|
this.credentials = getComputeCredential();
|
50
|
+
} else if(authMethod.toLowerCase().equals("json_key")) {
|
51
|
+
this.credentials = getServiceAccountCredentialFromJsonFile();
|
46
52
|
} else {
|
47
53
|
this.credentials = getServiceAccountCredential();
|
48
54
|
}
|
@@ -69,6 +75,14 @@ public class BigqueryAuthentication
|
|
69
75
|
.build();
|
70
76
|
}
|
71
77
|
|
78
|
+
private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
|
79
|
+
{
|
80
|
+
FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
|
81
|
+
|
82
|
+
return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
|
83
|
+
.createScoped(Collections.singleton(BigqueryScopes.BIGQUERY));
|
84
|
+
}
|
85
|
+
|
72
86
|
/**
|
73
87
|
* @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
|
74
88
|
* @see https://developers.google.com/accounts/docs/OAuth2
|
@@ -55,6 +55,10 @@ public class BigqueryOutputPlugin
|
|
55
55
|
Optional<LocalFile> getP12Keyfile();
|
56
56
|
void setP12Keyfile(Optional<LocalFile> p12Keyfile);
|
57
57
|
|
58
|
+
@Config("json_keyfile")
|
59
|
+
@ConfigDefault("null")
|
60
|
+
Optional<LocalFile> getJsonKeyfile();
|
61
|
+
|
58
62
|
@Config("application_name")
|
59
63
|
@ConfigDefault("\"Embulk BigQuery plugin\"")
|
60
64
|
String getApplicationName();
|
@@ -168,11 +172,22 @@ public class BigqueryOutputPlugin
|
|
168
172
|
}
|
169
173
|
}
|
170
174
|
|
175
|
+
if (task.getAuthMethod().getString().equals("json_key")) {
|
176
|
+
if (!task.getJsonKeyfile().isPresent()) {
|
177
|
+
throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
|
178
|
+
}
|
179
|
+
} else if (task.getAuthMethod().getString().equals("private_key")) {
|
180
|
+
if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
|
181
|
+
throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
|
182
|
+
}
|
183
|
+
}
|
184
|
+
|
171
185
|
try {
|
172
186
|
bigQueryWriter = new BigqueryWriter.Builder (
|
173
187
|
task.getAuthMethod().getString(),
|
174
188
|
task.getServiceAccountEmail(),
|
175
189
|
task.getP12Keyfile().transform(localFileToPathString()),
|
190
|
+
task.getJsonKeyfile().transform(localFileToPathString()),
|
176
191
|
task.getApplicationName())
|
177
192
|
.setAutoCreateTable(task.getAutoCreateTable())
|
178
193
|
.setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
|
@@ -357,7 +372,8 @@ public class BigqueryOutputPlugin
|
|
357
372
|
public enum AuthMethod
|
358
373
|
{
|
359
374
|
private_key("private_key"),
|
360
|
-
compute_engine("compute_engine")
|
375
|
+
compute_engine("compute_engine"),
|
376
|
+
json_key("json_key");
|
361
377
|
|
362
378
|
private final String string;
|
363
379
|
|
@@ -72,7 +72,10 @@ public class BigqueryWriter
|
|
72
72
|
this.ignoreUnknownValues = builder.ignoreUnknownValues;
|
73
73
|
this.allowQuotedNewlines = builder.allowQuotedNewlines;
|
74
74
|
|
75
|
-
BigqueryAuthentication auth = new BigqueryAuthentication(
|
75
|
+
BigqueryAuthentication auth = new BigqueryAuthentication(
|
76
|
+
builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath,
|
77
|
+
builder.jsonKeyFilePath, builder.applicationName
|
78
|
+
);
|
76
79
|
this.bigQueryClient = auth.getBigqueryClient();
|
77
80
|
|
78
81
|
if (autoCreateTable) {
|
@@ -336,6 +339,7 @@ public class BigqueryWriter
|
|
336
339
|
private final String authMethod;
|
337
340
|
private Optional<String> serviceAccountEmail;
|
338
341
|
private Optional<String> p12KeyFilePath;
|
342
|
+
private Optional<String> jsonKeyFilePath;
|
339
343
|
private String applicationName;
|
340
344
|
private boolean autoCreateTable;
|
341
345
|
private Optional<String> schemaPath;
|
@@ -350,11 +354,13 @@ public class BigqueryWriter
|
|
350
354
|
private boolean ignoreUnknownValues;
|
351
355
|
private boolean allowQuotedNewlines;
|
352
356
|
|
353
|
-
public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath,
|
357
|
+
public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath,
|
358
|
+
Optional<String> jsonKeyFilePath, String applicationName)
|
354
359
|
{
|
355
360
|
this.authMethod = authMethod;
|
356
361
|
this.serviceAccountEmail = serviceAccountEmail;
|
357
362
|
this.p12KeyFilePath = p12KeyFilePath;
|
363
|
+
this.jsonKeyFilePath = jsonKeyFilePath;
|
358
364
|
this.applicationName = applicationName;
|
359
365
|
}
|
360
366
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-10-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- ChangeLog
|
49
50
|
- README.md
|
50
51
|
- build.gradle
|
51
52
|
- gradle/wrapper/gradle-wrapper.jar
|
@@ -62,7 +63,7 @@ files:
|
|
62
63
|
- src/test/java/org/embulk/output/TestBigqueryWriter.java
|
63
64
|
- classpath/commons-codec-1.3.jar
|
64
65
|
- classpath/commons-logging-1.1.1.jar
|
65
|
-
- classpath/embulk-output-bigquery-0.1.
|
66
|
+
- classpath/embulk-output-bigquery-0.1.10.jar
|
66
67
|
- classpath/google-api-client-1.20.0.jar
|
67
68
|
- classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
|
68
69
|
- classpath/google-http-client-1.20.0.jar
|
Binary file
|