embulk-input-gcs 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c1337868e0a547834610ae572d49228a3394aff7
-  data.tar.gz: 52e83dd8d9b91eb7ca715ef36b3d74f8241e2860
+  metadata.gz: 24b44f4db77ef29422e03052b58cdf67378d5a79
+  data.tar.gz: 031a0686510df86a6b87780aa0da707cd7cd64cc
 SHA512:
-  metadata.gz: b8980edca632680de44f5311b3e2a3c622d94e3ee15f76eade05a536eb96cf64397e8916b4b47516a0724b9a2d2b03666e78ff2bc9bdee60352a6714e3d3383e
-  data.tar.gz: 8ad0624eb9576e37dbeac7dc2eead16bc72bfae8f51274d81042f0657821024ea2595ccd71f637fa9c63af049dac6bdd61fc9fe5df24ec8d1b36ee3bfff85a56
+  metadata.gz: eb5815752627aeef5c32de944d23e78c03313970e574f61e3706808bb4314643311e4d581ae2fc7589ede24e80ff53236cdd0defecbd647a2d382d26b3630dcc
+  data.tar.gz: 5ce214a4e252f78bbc9a9215d3edcc0de671220e2f5f9697687e4b4ac0d9d05303a5a724f399ad1a8175a8a8d85bd50d977fc01af8a718e74cb50981e34b3b6d

data/ChangeLog

@@ -0,0 +1,22 @@
+Release 0.1.7 - 2015-10-06
+
+* Added new auth method - json_keyfile of GCP(Google Cloud Platform)'s service account
+* Supported mapreduce-executor
+
+Release 0.1.6 - 2015-09-05
+
+* Added new auth method - pre-defined access token of GCE(Google Compute Engine)
+
+Release 0.1.5 - 2015-08-19
+
+* Upgraded embulk version to 0.7.0
+* Refactored
+
+Release 0.1.4 - 2015-06-27
+
+* Keep last last_path when input files is empty. @frsyuki thanks!
+* Refactored error handling logics.
+
+Release 0.1.3 - 2015-03-16
+
+* Changed supported Java version from 8 to 7

data/README.md

@@ -16,7 +16,7 @@ embulk gem install embulk-input-gcs
 
 ### Google Service Account Settings
 
-If you chose "private_key" as [auth_method](#Authentication), you can get service_account_email and private_key like below.
+If you chose "private_key" or "json_key" as [auth_method](#Authentication), you can get service_account_email and private_key or json_key like below.
 
 1. Make project at [Google Developers Console](https://console.developers.google.com/project).
 
@@ -26,9 +26,7 @@ If you chose "private_key" as [auth_method](#Authentication), you can get servic
 
     embulk-input-gcs can run "read-only" scopes.
 
-1. Generate private key in P12(PKCS12) format, and upload to machine.
-
-1. Write "EMAIL_ADDRESS" and fullpath of PKCS12 private key in yaml.
+1. Generate private key in P12(PKCS12) format or json_key, and upload to machine.
 
 ### run
 
@@ -40,9 +38,10 @@ embulk run /path/to/config.yml
 
 - **bucket** Google Cloud Storage bucket name (string, required)
 - **path_prefix** prefix of target keys (string, required)
-- **auth_method**  (string, optional, "private_key" or "compute_engine". default value is "private_key")
-- **service_account_email** Google Cloud Storage service_account_email (string, required)
-- **p12_keyfile_fullpath** fullpath of p12 key (string, required)
+- **auth_method**  (string, optional, "private_key", "json_key" or "compute_engine". default value is "private_key")
+- **service_account_email** Google Cloud Storage service_account_email (string, required when auth_method is private_key)
+- **p12_keyfile** fullpath of p12 key (string, required when auth_method is private_key)
+- **json_keyfile** fullpath of json_key (string, required when auth_method is json_key)
 - **application_name** application name anything you like (string, optional)
 
 ## Example
@@ -54,7 +53,7 @@ in:
   path_prefix: logs/csv-
   auth_method: private_key #default
   service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
-  p12_keyfile_path: /path/to/p12_keyfile.p12
+  p12_keyfile: /path/to/p12_keyfile.p12
   application_name: Anything you like
 ```
 
@@ -67,7 +66,7 @@ in:
   path_prefix: sample_
   auth_method: private_key #default
   service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
-  p12_keyfile_path: /path/to/p12_keyfile.p12
+  p12_keyfile: /path/to/p12_keyfile.p12
   application_name: Anything you like
   decoders:
   - {type: gzip}
@@ -89,23 +88,61 @@ out: {type: stdout}
 
 ## Authentication
 
-There are two methods supported to fetch access token for the service account.
+There are three methods supported to fetch access token for the service account.
+
+1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
+2. JSON key of GCP(Google Cloud Platform)'s service account
+3. Pre-defined access token (Google Compute Engine only)
+
+### Public-Private key pair of GCP's service account
+
+You first need to create a service account (client ID), download its private key and deploy the key with embulk.
+
+```yaml
+in:
+  type: gcs
+  auth_method: private_key
+  service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
+  p12_keyfile: /path/to/p12_keyfile.p12
+```
 
-1. Public-Private key pair
-2. Pre-defined access token (Compute Engine only)
+### JSON key of GCP's service account
 
-The examples above use the first one.  You first need to create a service account (client ID),
-download its private key and deploy the key with embulk.
+You first need to create a service account (client ID), download its json key and deploy the key with embulk.
+
+```yaml
+in:
+  type: gcs
+  auth_method: json_key
+  json_keyfile: /path/to/json_keyfile.json
+```
+
+You can also embed contents of json_keyfile at config.yml.
+
+```yaml
+in:
+  type: gcs
+  auth_method: json_key
+  json_keyfile:
+    content: |
+      {
+          "private_key_id": "123456789",
+          "private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
+          "client_email": "..."
+       }
+```
+
+### Pre-defined access token(GCE only)
 
 On the other hand, you don't need to explicitly create a service account for embulk when you
-run embulk in Google Compute Engine. In this second authentication method, you need to
-add the API scope "https://www.googleapis.com/auth/devstorage.read_only" to the scope list of your
-Compute Engine instance, then you can configure embulk like this.
+run embulk in Google Compute Engine. In this third authentication method, you need to
+add the API scope "https://www.googleapis.com/auth/bigquery" to the scope list of your
+Compute Engine VM instance, then you can configure embulk like this.
 
 [Setting the scope of service account access for instances](https://cloud.google.com/compute/docs/authentication)
 
 ```yaml
-input:
+in:
   type: gcs
   auth_method: compute_engine
 ```

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.6"
+version = "0.1.7"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.1"

data/src/main/java/org/embulk/input/gcs/GcsAuthentication.java

@@ -1,11 +1,13 @@
 package org.embulk.input.gcs;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import java.security.GeneralSecurityException;
+import java.util.Collections;
 
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.googleapis.compute.ComputeCredential;
@@ -26,16 +28,19 @@ public class GcsAuthentication
     private final Logger log = Exec.getLogger(GcsAuthentication.class);
     private final Optional<String> serviceAccountEmail;
     private final Optional<String> p12KeyFilePath;
+    private final Optional<String> jsonKeyFilePath;
     private final String applicationName;
     private final HttpTransport httpTransport;
     private final JsonFactory jsonFactory;
     private final HttpRequestInitializer credentials;
 
-    public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
+    public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
+            Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
             throws IOException, GeneralSecurityException
     {
         this.serviceAccountEmail = serviceAccountEmail;
         this.p12KeyFilePath = p12KeyFilePath;
+        this.jsonKeyFilePath = jsonKeyFilePath;
         this.applicationName = applicationName;
 
         this.httpTransport = GoogleNetHttpTransport.newTrustedTransport();
@@ -43,6 +48,8 @@ public class GcsAuthentication
 
         if (authMethod.equals("compute_engine")) {
             this.credentials = getComputeCredential();
+        } else if(authMethod.toLowerCase().equals("json_key")) {
+            this.credentials = getServiceAccountCredentialFromJsonFile();
         } else {
             this.credentials = getServiceAccountCredential();
         }
@@ -69,6 +76,14 @@ public class GcsAuthentication
                 .build();
     }
 
+    private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
+    {
+        FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
+
+        return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
+                .createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_ONLY));
+    }
+
     /**
      * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
      * @see https://developers.google.com/accounts/docs/OAuth2

data/src/main/java/org/embulk/input/gcs/GcsFileInputPlugin.java

@@ -6,8 +6,11 @@ import java.util.Collections;
 import java.io.IOException;
 import java.io.InputStream;
 import java.math.BigInteger;
+
 import com.google.common.collect.ImmutableList;
 import com.google.common.base.Optional;
+import com.google.common.base.Function;
+import com.google.common.base.Throwables;
 import java.security.GeneralSecurityException;
 
 import org.embulk.config.TaskReport;
@@ -23,6 +26,7 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
+import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.util.InputStreamFileInput;
 
 import org.slf4j.Logger;
@@ -60,10 +64,20 @@ public class GcsFileInputPlugin
         @ConfigDefault("\"Embulk GCS input plugin\"")
         String getApplicationName();
 
+        // kept for backward compatibility
         @Config("p12_keyfile_fullpath")
         @ConfigDefault("null")
         Optional<String> getP12KeyfileFullpath();
 
+        @Config("p12_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getP12Keyfile();
+        void setP12Keyfile(Optional<LocalFile> p12Keyfile);
+
+        @Config("json_keyfile")
+        @ConfigDefault("null")
+        Optional<LocalFile> getJsonKeyfile();
+
         List<String> getFiles();
         void setFiles(List<String> files);
 
@@ -72,6 +86,7 @@ public class GcsFileInputPlugin
     }
 
     private static final Logger log = Exec.getLogger(GcsFileInputPlugin.class);
+    private static GcsAuthentication auth;
 
     @Override
     public ConfigDiff transaction(ConfigSource config,
@@ -79,6 +94,39 @@ public class GcsFileInputPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        if (task.getP12KeyfileFullpath().isPresent()) {
+            if (task.getP12Keyfile().isPresent()) {
+                throw new ConfigException("Setting both p12_keyfile_fullpath and p12_keyfile is invalid");
+            }
+            try {
+                task.setP12Keyfile(Optional.of(LocalFile.of(task.getP12KeyfileFullpath().get())));
+            } catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
+        }
+
+        if (task.getAuthMethod().getString().equals("json_key")) {
+            if (!task.getJsonKeyfile().isPresent()) {
+                throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
+            }
+        } else if (task.getAuthMethod().getString().equals("private_key")) {
+            if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
+                throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
+            }
+        }
+
+        try {
+            auth = new GcsAuthentication(
+                    task.getAuthMethod().getString(),
+                    task.getServiceAccountEmail(),
+                    task.getP12Keyfile().transform(localFileToPathString()),
+                    task.getJsonKeyfile().transform(localFileToPathString()),
+                    task.getApplicationName()
+            );
+        } catch (GeneralSecurityException | IOException ex) {
+            throw new ConfigException(ex);
+        }
+
         // list files recursively
         task.setFiles(listFiles(task));
         // number of processors is same with number of files
@@ -117,18 +165,29 @@ public class GcsFileInputPlugin
     {
     }
 
-    private static Storage newGcsClient(final PluginTask task) {
+    private static Storage newGcsClient(final PluginTask task)
+    {
         Storage client = null;
         try {
-            GcsAuthentication auth = new GcsAuthentication(task.getAuthMethod().getString(), task.getServiceAccountEmail(), task.getP12KeyfileFullpath(), task.getApplicationName());
             client = auth.getGcsClient(task.getBucket());
-        } catch (GeneralSecurityException | IOException ex) {
+        } catch (IOException ex) {
             throw new ConfigException(ex);
         }
 
         return client;
     }
 
+    private Function<LocalFile, String> localFileToPathString()
+    {
+        return new Function<LocalFile, String>()
+        {
+            public String apply(LocalFile file)
+            {
+                return file.getPath().toString();
+            }
+        };
+    }
+
     public List<String> listFiles(PluginTask task)
     {
         Storage client = newGcsClient(task);
@@ -259,7 +318,8 @@ public class GcsFileInputPlugin
     public enum AuthMethod
     {
         private_key("private_key"),
-        compute_engine("compute_engine");
+        compute_engine("compute_engine"),
+        json_key("json_key");
 
         private final String string;
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-05 00:00:00.000000000 Z
+date: 2015-10-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- ChangeLog
 - README.md
 - build.gradle
 - gradle/wrapper/gradle-wrapper.jar
@@ -59,7 +60,7 @@ files:
 - src/test/java/org/embulk/input/gcs/TestGcsFileInputPlugin.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-input-gcs-0.1.6.jar
+- classpath/embulk-input-gcs-0.1.7.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev27-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar