embulk-output-bigquery 0.1.9 → 0.1.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 01ec741b2dec7dc95c1ae29254e3389a7af28beb
4
- data.tar.gz: 230b0712b540a250da355252437d9f4df118ac7a
3
+ metadata.gz: a87dead2fa7962e1c2b07c86adc52939c2ed4205
4
+ data.tar.gz: 34db5af6375ce61615b5aeafb94cf9eb11e576c0
5
5
  SHA512:
6
- metadata.gz: d7cb7461e6b607e55c8379b65c6e2886a9249619177f8a2ab49ab0e98e44fb9db0054407e27cd52a24f923f1dc246a5f20897da16cf216bd6c8905637fb4353e
7
- data.tar.gz: cda8c8537c0e34737c0f81fea0118e9fa61d0330fd8a8d1b16fd75e273587b6b1808a7590a1e4a322ebffe4ef4a5d4c002ecc02bd4df7114db090b08e3976eae
6
+ metadata.gz: 46ce7c4c2a5b5e1213724bf571bdf3764320d34cc787f79bc6ef20cef4681f8a402c600bdef807311f550045ce0c95f5a6eb5abc4262fce746705f86119ccc76
7
+ data.tar.gz: bc5bfb7050882ef5312a78ee6c0d746e76bae0045baa0c82cd9f4e2c67754aeccc834ce1b7864ad8e105d7e56f2d3eec63131c7c0feee7a79dd0f18b24293611
data/ChangeLog ADDED
@@ -0,0 +1,42 @@
1
+ Release 0.1.10 - 2015-10-06
2
+
3
+ * Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account
4
+
5
+ Release 0.1.9 - 2015-08-19
6
+
7
+ * Upgraded embulk version to 0.7.1
8
+
9
+ Release 0.1.8 - 2015-08-19
10
+
11
+ * Supported mapreduce-executor. @frsyuki thanks!
12
+ * Fixed job_id generation logic
13
+ * Refactored
14
+
15
+ Release 0.1.7 - 2015-03-20
16
+
17
+ * Added allow_quoted_newlines option
18
+ * Upgraded embulk version to 0.6.8
19
+
20
+ Release 0.1.6 - 2015-04-23
21
+
22
+ * Added ignore_unknown_values option to job_id generation logic.
23
+
24
+ Release 0.1.5 - 2015-04-23
25
+
26
+ * Added ignore_unknown_values option. @takus thanks!
27
+
28
+ Release 0.1.4 - 2015-04-21
29
+
30
+ * Added prevent_duplicate_insert option
31
+
32
+ Release 0.1.3 - 2015-04-06
33
+
34
+ * Added new auth method - pre-defined access token of GCE(Google Compute Engine)
35
+ * Updated Google provided libraries
36
+ * http-client:google-http-client-jackson2 from 1.19.0 to 1.20.0
37
+ * apis:google-api-services-bigquery from v2-rev193-1.19.1 to v2-rev205-1.20.0
38
+
39
+ Release 0.1.2 - 2015-04-01
40
+
41
+ * Changed bulk-load method from "via GCS" to direct-insert
42
+ * added dynamic table creationg option
data/README.md CHANGED
@@ -28,9 +28,10 @@ OAuth flow for installed applications.
28
28
 
29
29
  | name | type | required? | default | description |
30
30
  |:--------------------------|:------------|:-----------|:-------------|:-----------------------|
31
- | auth_method | string | optional | "private_key" | `private_key` or `compute_engine`
31
+ | auth_method | string | optional | "private_key" | `private_key` , `json_key` or `compute_engine`
32
32
  | service_account_email | string | required when auth_method is private_key | | Your Google service account email
33
33
  | p12_keyfile | string | required when auth_method is private_key | | Fullpath of private key in P12(PKCS12) format |
34
+ | json_keyfile | string | required when auth_method is json_key | | Fullpath of json key |
34
35
  | sequence_format | string | optional | %03d.%02d | |
35
36
  | file_ext | string | optional | | e.g. ".csv.gz" ".json.gz" |
36
37
  | project | string | required | | project_id |
@@ -81,18 +82,58 @@ out:
81
82
 
82
83
  ### Authentication
83
84
 
84
- There are two methods supported to fetch access token for the service account.
85
+ There are three methods supported to fetch access token for the service account.
85
86
 
86
- 1. Public-Private key pair
87
- 2. Predefined access token (Compute Engine only)
87
+ 1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
88
+ 2. JSON key of GCP(Google Cloud Platform)'s service account
89
+ 3. Pre-defined access token (Google Compute Engine only)
88
90
 
89
- The examples above use the first one. You first need to create a service account (client ID),
91
+ #### Public-Private key pair of GCP's service account
92
+
93
+ You first need to create a service account (client ID),
90
94
  download its private key and deploy the key with embulk.
91
95
 
96
+ ```yaml
97
+ out:
98
+ type: bigquery
99
+ auth_method: private_key # default
100
+ service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
101
+ p12_keyfile: /path/to/p12_keyfile.p12
102
+ ```
103
+
104
+ #### JSON key of GCP's service account
105
+
106
+ You first need to create a service account (client ID),
107
+ download its json key and deploy the key with embulk.
108
+
109
+ ```yaml
110
+ out:
111
+ type: bigquery
112
+ auth_method: json_key
113
+ json_keyfile: /path/to/json_keyfile.json
114
+ ```
115
+
116
+ You can also embed contents of json_keyfile at config.yml.
117
+
118
+ ```yaml
119
+ in:
120
+ type: gcs
121
+ auth_method: json_key
122
+ json_keyfile:
123
+ content: |
124
+ {
125
+ "private_key_id": "123456789",
126
+ "private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
127
+ "client_email": "..."
128
+ }
129
+ ```
130
+
131
+ #### Pre-defined access token(GCE only)
132
+
92
133
  On the other hand, you don't need to explicitly create a service account for embulk when you
93
- run embulk in Google Compute Engine. In this second authentication method, you need to
134
+ run embulk in Google Compute Engine. In this third authentication method, you need to
94
135
  add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
95
- Compute Engine instance, then you can configure embulk like this.
136
+ Compute Engine VM instance, then you can configure embulk like this.
96
137
 
97
138
  ```yaml
98
139
  out:
data/build.gradle CHANGED
@@ -15,7 +15,7 @@ configurations {
15
15
  sourceCompatibility = 1.7
16
16
  targetCompatibility = 1.7
17
17
 
18
- version = "0.1.9"
18
+ version = "0.1.10"
19
19
 
20
20
  dependencies {
21
21
  compile "org.embulk:embulk-core:0.7.1"
@@ -1,12 +1,13 @@
1
1
  package org.embulk.output;
2
2
 
3
3
  import java.io.File;
4
+ import java.io.FileInputStream;
4
5
  import java.io.IOException;
6
+ import java.util.Collections;
7
+ import java.security.GeneralSecurityException;
5
8
 
6
9
  import com.google.common.base.Optional;
7
10
  import com.google.common.collect.ImmutableList;
8
- import java.security.GeneralSecurityException;
9
-
10
11
  import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
11
12
  import com.google.api.client.googleapis.compute.ComputeCredential;
12
13
  import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
@@ -26,16 +27,19 @@ public class BigqueryAuthentication
26
27
  private final Logger log = Exec.getLogger(BigqueryAuthentication.class);
27
28
  private final Optional<String> serviceAccountEmail;
28
29
  private final Optional<String> p12KeyFilePath;
30
+ private final Optional<String> jsonKeyFilePath;
29
31
  private final String applicationName;
30
32
  private final HttpTransport httpTransport;
31
33
  private final JsonFactory jsonFactory;
32
34
  private final HttpRequestInitializer credentials;
33
35
 
34
- public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
36
+ public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail,
37
+ Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
35
38
  throws IOException, GeneralSecurityException
36
39
  {
37
40
  this.serviceAccountEmail = serviceAccountEmail;
38
41
  this.p12KeyFilePath = p12KeyFilePath;
42
+ this.jsonKeyFilePath = jsonKeyFilePath;
39
43
  this.applicationName = applicationName;
40
44
 
41
45
  this.httpTransport = GoogleNetHttpTransport.newTrustedTransport();
@@ -43,6 +47,8 @@ public class BigqueryAuthentication
43
47
 
44
48
  if (authMethod.toLowerCase().equals("compute_engine")) {
45
49
  this.credentials = getComputeCredential();
50
+ } else if(authMethod.toLowerCase().equals("json_key")) {
51
+ this.credentials = getServiceAccountCredentialFromJsonFile();
46
52
  } else {
47
53
  this.credentials = getServiceAccountCredential();
48
54
  }
@@ -69,6 +75,14 @@ public class BigqueryAuthentication
69
75
  .build();
70
76
  }
71
77
 
78
+ private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
79
+ {
80
+ FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
81
+
82
+ return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
83
+ .createScoped(Collections.singleton(BigqueryScopes.BIGQUERY));
84
+ }
85
+
72
86
  /**
73
87
  * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
74
88
  * @see https://developers.google.com/accounts/docs/OAuth2
@@ -55,6 +55,10 @@ public class BigqueryOutputPlugin
55
55
  Optional<LocalFile> getP12Keyfile();
56
56
  void setP12Keyfile(Optional<LocalFile> p12Keyfile);
57
57
 
58
+ @Config("json_keyfile")
59
+ @ConfigDefault("null")
60
+ Optional<LocalFile> getJsonKeyfile();
61
+
58
62
  @Config("application_name")
59
63
  @ConfigDefault("\"Embulk BigQuery plugin\"")
60
64
  String getApplicationName();
@@ -168,11 +172,22 @@ public class BigqueryOutputPlugin
168
172
  }
169
173
  }
170
174
 
175
+ if (task.getAuthMethod().getString().equals("json_key")) {
176
+ if (!task.getJsonKeyfile().isPresent()) {
177
+ throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
178
+ }
179
+ } else if (task.getAuthMethod().getString().equals("private_key")) {
180
+ if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
181
+ throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
182
+ }
183
+ }
184
+
171
185
  try {
172
186
  bigQueryWriter = new BigqueryWriter.Builder (
173
187
  task.getAuthMethod().getString(),
174
188
  task.getServiceAccountEmail(),
175
189
  task.getP12Keyfile().transform(localFileToPathString()),
190
+ task.getJsonKeyfile().transform(localFileToPathString()),
176
191
  task.getApplicationName())
177
192
  .setAutoCreateTable(task.getAutoCreateTable())
178
193
  .setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
@@ -357,7 +372,8 @@ public class BigqueryOutputPlugin
357
372
  public enum AuthMethod
358
373
  {
359
374
  private_key("private_key"),
360
- compute_engine("compute_engine");
375
+ compute_engine("compute_engine"),
376
+ json_key("json_key");
361
377
 
362
378
  private final String string;
363
379
 
@@ -72,7 +72,10 @@ public class BigqueryWriter
72
72
  this.ignoreUnknownValues = builder.ignoreUnknownValues;
73
73
  this.allowQuotedNewlines = builder.allowQuotedNewlines;
74
74
 
75
- BigqueryAuthentication auth = new BigqueryAuthentication(builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath, builder.applicationName);
75
+ BigqueryAuthentication auth = new BigqueryAuthentication(
76
+ builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath,
77
+ builder.jsonKeyFilePath, builder.applicationName
78
+ );
76
79
  this.bigQueryClient = auth.getBigqueryClient();
77
80
 
78
81
  if (autoCreateTable) {
@@ -336,6 +339,7 @@ public class BigqueryWriter
336
339
  private final String authMethod;
337
340
  private Optional<String> serviceAccountEmail;
338
341
  private Optional<String> p12KeyFilePath;
342
+ private Optional<String> jsonKeyFilePath;
339
343
  private String applicationName;
340
344
  private boolean autoCreateTable;
341
345
  private Optional<String> schemaPath;
@@ -350,11 +354,13 @@ public class BigqueryWriter
350
354
  private boolean ignoreUnknownValues;
351
355
  private boolean allowQuotedNewlines;
352
356
 
353
- public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
357
+ public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath,
358
+ Optional<String> jsonKeyFilePath, String applicationName)
354
359
  {
355
360
  this.authMethod = authMethod;
356
361
  this.serviceAccountEmail = serviceAccountEmail;
357
362
  this.p12KeyFilePath = p12KeyFilePath;
363
+ this.jsonKeyFilePath = jsonKeyFilePath;
358
364
  this.applicationName = applicationName;
359
365
  }
360
366
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-bigquery
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.9
4
+ version: 0.1.10
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-19 00:00:00.000000000 Z
11
+ date: 2015-10-06 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
48
  - .gitignore
49
+ - ChangeLog
49
50
  - README.md
50
51
  - build.gradle
51
52
  - gradle/wrapper/gradle-wrapper.jar
@@ -62,7 +63,7 @@ files:
62
63
  - src/test/java/org/embulk/output/TestBigqueryWriter.java
63
64
  - classpath/commons-codec-1.3.jar
64
65
  - classpath/commons-logging-1.1.1.jar
65
- - classpath/embulk-output-bigquery-0.1.9.jar
66
+ - classpath/embulk-output-bigquery-0.1.10.jar
66
67
  - classpath/google-api-client-1.20.0.jar
67
68
  - classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
68
69
  - classpath/google-http-client-1.20.0.jar