embulk-output-bigquery 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01ec741b2dec7dc95c1ae29254e3389a7af28beb
-  data.tar.gz: 230b0712b540a250da355252437d9f4df118ac7a
+  metadata.gz: a87dead2fa7962e1c2b07c86adc52939c2ed4205
+  data.tar.gz: 34db5af6375ce61615b5aeafb94cf9eb11e576c0
 SHA512:
-  metadata.gz: d7cb7461e6b607e55c8379b65c6e2886a9249619177f8a2ab49ab0e98e44fb9db0054407e27cd52a24f923f1dc246a5f20897da16cf216bd6c8905637fb4353e
-  data.tar.gz: cda8c8537c0e34737c0f81fea0118e9fa61d0330fd8a8d1b16fd75e273587b6b1808a7590a1e4a322ebffe4ef4a5d4c002ecc02bd4df7114db090b08e3976eae
+  metadata.gz: 46ce7c4c2a5b5e1213724bf571bdf3764320d34cc787f79bc6ef20cef4681f8a402c600bdef807311f550045ce0c95f5a6eb5abc4262fce746705f86119ccc76
+  data.tar.gz: bc5bfb7050882ef5312a78ee6c0d746e76bae0045baa0c82cd9f4e2c67754aeccc834ce1b7864ad8e105d7e56f2d3eec63131c7c0feee7a79dd0f18b24293611

data/ChangeLog

@@ -0,0 +1,42 @@
+Release 0.1.10 - 2015-10-06
+
+* Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account
+
+Release 0.1.9 - 2015-08-19
+
+* Upgraded embulk version to 0.7.1
+
+Release 0.1.8 - 2015-08-19
+
+* Supported mapreduce-executor. @frsyuki thanks!
+* Fixed job_id generation logic
+* Refactored
+
+Release 0.1.7 - 2015-03-20
+
+* Added allow_quoted_newlines option
+* Upgraded embulk version to 0.6.8
+
+Release 0.1.6 - 2015-04-23
+
+* Added ignore_unknown_values option to job_id generation logic.
+
+Release 0.1.5 - 2015-04-23
+
+* Added ignore_unknown_values option. @takus thanks!
+
+Release 0.1.4 - 2015-04-21
+
+* Added prevent_duplicate_insert option
+
+Release 0.1.3 - 2015-04-06
+
+* Added new auth method - pre-defined access token of GCE(Google Compute Engine)
+* Updated Google provided libraries
+  * http-client:google-http-client-jackson2 from 1.19.0 to 1.20.0
+  * apis:google-api-services-bigquery from v2-rev193-1.19.1 to v2-rev205-1.20.0
+
+Release 0.1.2 - 2015-04-01
+
+* Changed bulk-load method from "via GCS" to direct-insert
+* added dynamic table creationg option

data/README.md

@@ -28,9 +28,10 @@ OAuth flow for installed applications.
 
 | name                      | type        | required?  | default      | description            |  
 |:--------------------------|:------------|:-----------|:-------------|:-----------------------|
-|  auth_method              | string      | optional   | "private_key"  | `private_key` or `compute_engine`
+|  auth_method              | string      | optional   | "private_key"  | `private_key` , `json_key` or `compute_engine`
 |  service_account_email    | string      | required when auth_method is private_key  |   | Your Google service account email
 |  p12_keyfile              | string      | required when auth_method is private_key   |   | Fullpath of private key in P12(PKCS12) format |
+|  json_keyfile             | string      | required when auth_method is json_key     |   | Fullpath of json key |
 |  sequence_format          | string      | optional   | %03d.%02d      |  |
 |  file_ext                 | string      | optional   |                | e.g. ".csv.gz" ".json.gz" |
 |  project                  | string      | required   |                | project_id |
@@ -81,18 +82,58 @@ out:
 
 ### Authentication
 
-There are two methods supported to fetch access token for the service account.
+There are three methods supported to fetch access token for the service account.
 
-1. Public-Private key pair
-2. Predefined access token (Compute Engine only)
+1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
+2. JSON key of GCP(Google Cloud Platform)'s service account
+3. Pre-defined access token (Google Compute Engine only)
 
-The examples above use the first one.  You first need to create a service account (client ID),
+#### Public-Private key pair of GCP's service account
+
+You first need to create a service account (client ID),
 download its private key and deploy the key with embulk.
 
+```yaml
+out:
+  type: bigquery
+  auth_method: private_key   # default
+  service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
+  p12_keyfile: /path/to/p12_keyfile.p12
+```
+
+#### JSON key of GCP's service account
+
+You first need to create a service account (client ID),
+download its json key and deploy the key with embulk.
+
+```yaml
+out:
+  type: bigquery
+  auth_method: json_key
+  json_keyfile: /path/to/json_keyfile.json
+```
+
+You can also embed contents of json_keyfile at config.yml.
+
+```yaml
+in:
+  type: gcs
+  auth_method: json_key
+  json_keyfile:
+    content: |
+      {
+          "private_key_id": "123456789",
+          "private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
+          "client_email": "..."
+       }
+```
+
+#### Pre-defined access token(GCE only)
+
 On the other hand, you don't need to explicitly create a service account for embulk when you
-run embulk in Google Compute Engine.  In this second authentication method, you need to
+run embulk in Google Compute Engine. In this third authentication method, you need to
 add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
-Compute Engine instance, then you can configure embulk like this.
+Compute Engine VM instance, then you can configure embulk like this.
 
 ```yaml
 out:

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.9"
+version = "0.1.10"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.1"

data/src/main/java/org/embulk/output/BigqueryAuthentication.java

@@ -1,12 +1,13 @@
 package org.embulk.output;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
+import java.util.Collections;
+import java.security.GeneralSecurityException;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-import java.security.GeneralSecurityException;
-
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.compute.ComputeCredential;
 import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
@@ -26,16 +27,19 @@ public class BigqueryAuthentication
     private final Logger log = Exec.getLogger(BigqueryAuthentication.class);
     private final Optional<String> serviceAccountEmail;
     private final Optional<String> p12KeyFilePath;
+    private final Optional<String> jsonKeyFilePath;
     private final String applicationName;
     private final HttpTransport httpTransport;
     private final JsonFactory jsonFactory;
     private final HttpRequestInitializer credentials;
 
-    public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
+    public BigqueryAuthentication(String authMethod, Optional<String> serviceAccountEmail,
+            Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
             throws IOException, GeneralSecurityException
     {
         this.serviceAccountEmail = serviceAccountEmail;
         this.p12KeyFilePath = p12KeyFilePath;
+        this.jsonKeyFilePath = jsonKeyFilePath;
         this.applicationName = applicationName;
 
         this.httpTransport = GoogleNetHttpTransport.newTrustedTransport();
@@ -43,6 +47,8 @@ public class BigqueryAuthentication
 
         if (authMethod.toLowerCase().equals("compute_engine")) {
             this.credentials = getComputeCredential();
+        } else if(authMethod.toLowerCase().equals("json_key")) {
+            this.credentials = getServiceAccountCredentialFromJsonFile();
         } else {
             this.credentials = getServiceAccountCredential();
         }
@@ -69,6 +75,14 @@ public class BigqueryAuthentication
                 .build();
     }
 
+    private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
+    {
+        FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
+
+        return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
+                .createScoped(Collections.singleton(BigqueryScopes.BIGQUERY));
+    }
+
     /**
      * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
      * @see https://developers.google.com/accounts/docs/OAuth2

data/src/main/java/org/embulk/output/BigqueryOutputPlugin.java

@@ -55,6 +55,10 @@ public class BigqueryOutputPlugin
         Optional<LocalFile> getP12Keyfile();
         void setP12Keyfile(Optional<LocalFile> p12Keyfile);
 
+        @Config("json_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getJsonKeyfile();
+
         @Config("application_name")
         @ConfigDefault("\"Embulk BigQuery plugin\"")
         String getApplicationName();
@@ -168,11 +172,22 @@ public class BigqueryOutputPlugin
             }
         }
 
+        if (task.getAuthMethod().getString().equals("json_key")) {
+            if (!task.getJsonKeyfile().isPresent()) {
+                throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
+            }
+        } else if (task.getAuthMethod().getString().equals("private_key")) {
+            if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
+                throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
+            }
+        }
+
         try {
             bigQueryWriter = new BigqueryWriter.Builder (
                     task.getAuthMethod().getString(),
                     task.getServiceAccountEmail(),
                     task.getP12Keyfile().transform(localFileToPathString()),
+                    task.getJsonKeyfile().transform(localFileToPathString()),
                     task.getApplicationName())
                     .setAutoCreateTable(task.getAutoCreateTable())
                     .setSchemaPath(task.getSchemaFile().transform(localFileToPathString()))
@@ -357,7 +372,8 @@ public class BigqueryOutputPlugin
     public enum AuthMethod
     {
         private_key("private_key"),
-        compute_engine("compute_engine");
+        compute_engine("compute_engine"),
+        json_key("json_key");
 
         private final String string;
 

data/src/main/java/org/embulk/output/BigqueryWriter.java

@@ -72,7 +72,10 @@ public class BigqueryWriter
         this.ignoreUnknownValues = builder.ignoreUnknownValues;
         this.allowQuotedNewlines = builder.allowQuotedNewlines;
 
-        BigqueryAuthentication auth = new BigqueryAuthentication(builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath, builder.applicationName);
+        BigqueryAuthentication auth = new BigqueryAuthentication(
+                builder.authMethod, builder.serviceAccountEmail, builder.p12KeyFilePath,
+                builder.jsonKeyFilePath, builder.applicationName
+        );
         this.bigQueryClient = auth.getBigqueryClient();
 
         if (autoCreateTable) {
@@ -336,6 +339,7 @@ public class BigqueryWriter
         private final String authMethod;
         private Optional<String> serviceAccountEmail;
         private Optional<String> p12KeyFilePath;
+        private Optional<String> jsonKeyFilePath;
         private String applicationName;
         private boolean autoCreateTable;
         private Optional<String> schemaPath;
@@ -350,11 +354,13 @@ public class BigqueryWriter
         private boolean ignoreUnknownValues;
         private boolean allowQuotedNewlines;
 
-        public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
+        public Builder(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath,
+                Optional<String> jsonKeyFilePath, String applicationName)
         {
             this.authMethod = authMethod;
             this.serviceAccountEmail = serviceAccountEmail;
             this.p12KeyFilePath = p12KeyFilePath;
+            this.jsonKeyFilePath = jsonKeyFilePath;
             this.applicationName = applicationName;
         }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-bigquery
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- ChangeLog
 - README.md
 - build.gradle
 - gradle/wrapper/gradle-wrapper.jar
@@ -62,7 +63,7 @@ files:
 - src/test/java/org/embulk/output/TestBigqueryWriter.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-bigquery-0.1.9.jar
+- classpath/embulk-output-bigquery-0.1.10.jar
 - classpath/google-api-client-1.20.0.jar
 - classpath/google-api-services-bigquery-v2-rev205-1.20.0.jar
 - classpath/google-http-client-1.20.0.jar