embulk-output-gcs 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7cc14163dab8e5bcae3c1b71d6a6e51493f460f3
-  data.tar.gz: 0ed896983dd82d5753d5beb57b9d10642c2521f7
+  metadata.gz: 94fccd956be721058fbf74ffd57017d05caf8e5b
+  data.tar.gz: e7e35ef7704225a633d47babdd21c91edd6ad272
 SHA512:
-  metadata.gz: 0d946fc00317318cc84605f589c6c9c5b5c6d8525b799f3a69e87d5f14d6c2060084c2e3a528a6a84dec8210bfe3bccf3a0ab9a639c1c1051f0ab83c30db34dd
-  data.tar.gz: f78b3ea4c04e04364787bfe987b4d5542e97ba3f41ed0c873a0142dea5b7dcf0bb5d9fe9b4c8db4e0ed649868517bc4e36ce88142b9a6e1b000f3e85e38f6491
+  metadata.gz: 2b2325cf0230d16f57fb7a247a9237428fd2d4527cd6848367a5989e3d8bc9a9da7f282b3a81ec97ad2425da9bb70cbd0a26465fcc1dd747ce3e804908cd48ad
+  data.tar.gz: ceb4828ba593c2e2ab8cf043bce26b4509867489cf3b7eac96fd00f1ef10b5ffe8895587a1844d7b01671ea0373cd30b82d39a94da2caf8183cdee206dcdd4e7

data/README.md

@@ -15,6 +15,7 @@ Google Cloud Storage output plugin for [Embulk](https://github.com/embulk/embulk
 - **path_prefix**: Prefix of output keys (string, required)
 - **file_ext**: Extention of output file (string, required)
 - **content_type**: content type of output file (string, optional, default value is "application/octet-stream")
+- **auth_method**: Authentication method `private_key` or `compute_engine` (string, optional, default value is "private_key")
 - **service_account_email**: Google Cloud Platform service account email (string, required)
 - **p12_keyfile_path**: Private key file fullpath of Google Cloud Platform service account (string, required)
 - **application_name**: Application name, anything you like (string, optional, default value is "embulk-output-gcs")
@@ -27,6 +28,7 @@ out:
   bucket: your-gcs-bucket-name
   path_prefix: logs/out
   file_ext: .csv
+  auth_method: `private_key` #default
   service_account_email: 'XYZ@developer.gserviceaccount.com'
   p12_keyfile_path: '/path/to/private/key.p12'
   formatter:
@@ -34,6 +36,29 @@ out:
     encoding: UTF-8
 ```
 
+## Authentication
+
+There are two methods supported to fetch access token for the service account.
+
+1. Public-Private key pair
+2. Pre-defined access token (Compute Engine only)
+
+The examples above use the first one.  You first need to create a service account (client ID),
+download its private key and deploy the key with embulk.
+
+On the other hand, you don't need to explicitly create a service account for embulk when you
+run embulk in Google Compute Engine. In this second authentication method, you need to
+add the API scope "https://www.googleapis.com/auth/devstorage.read_write" to the scope list of your
+Compute Engine instance, then you can configure embulk like this.
+
+[Setting the scope of service account access for instances](https://cloud.google.com/compute/docs/authentication)
+
+```yaml
+out:
+  type: gcs
+  auth_method: compute_engine
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -15,11 +15,11 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.1.1"
+version = "0.2.0"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.5.1"
-    provided "org.embulk:embulk-core:0.5.1"
+    compile  "org.embulk:embulk-core:0.7.3"
+    provided "org.embulk:embulk-core:0.7.3"
 
     compile "com.google.http-client:google-http-client-jackson2:1.19.0"
     compile ("com.google.apis:google-api-services-storage:v1-rev28-1.19.1") {exclude module: "guava-jdk5"}

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Wed Feb 04 13:46:12 PST 2015
+#Tue Aug 11 00:26:20 PDT 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/src/main/java/org/embulk/output/GcsAuthentication.java

@@ -0,0 +1,96 @@
+package org.embulk.output;
+
+import java.io.File;
+import java.io.IOException;
+
+import com.google.api.client.http.apache.ApacheHttpTransport;
+import com.google.api.services.storage.model.Objects;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import java.security.GeneralSecurityException;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+import com.google.api.client.googleapis.compute.ComputeCredential;
+import com.google.api.client.http.HttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.http.HttpRequestInitializer;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+import com.google.api.services.storage.Storage;
+import com.google.api.services.storage.StorageScopes;
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+public class GcsAuthentication
+{
+	private final Logger log = Exec.getLogger(GcsAuthentication.class);
+	private final Optional<String> serviceAccountEmail;
+	private final Optional<String> p12KeyFilePath;
+	private final String applicationName;
+	private final HttpTransport httpTransport;
+	private final JsonFactory jsonFactory;
+	private final HttpRequestInitializer credentials;
+
+	public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
+			throws IOException, GeneralSecurityException
+	{
+		this.serviceAccountEmail = serviceAccountEmail;
+		this.p12KeyFilePath = p12KeyFilePath;
+		this.applicationName = applicationName;
+
+		this.httpTransport = new ApacheHttpTransport.Builder().build();
+		this.jsonFactory = new JacksonFactory();
+
+		if (authMethod.equals("compute_engine")) {
+			this.credentials = getComputeCredential();
+		} else {
+			this.credentials = getServiceAccountCredential();
+		}
+	}
+
+	/**
+	 * @see https://developers.google.com/accounts/docs/OAuth2ServiceAccount#authorizingrequests
+	 */
+	private GoogleCredential getServiceAccountCredential() throws IOException, GeneralSecurityException
+	{
+		// @see https://cloud.google.com/compute/docs/api/how-tos/authorization
+		// @see https://developers.google.com/resources/api-libraries/documentation/storage/v1/java/latest/com/google/api/services/storage/STORAGE_SCOPE.html
+		// @see https://developers.google.com/resources/api-libraries/documentation/bigquery/v2/java/latest/com/google/api/services/bigquery/BigqueryScopes.html
+		return new GoogleCredential.Builder()
+				.setTransport(httpTransport)
+				.setJsonFactory(jsonFactory)
+				.setServiceAccountId(serviceAccountEmail.orNull())
+				.setServiceAccountScopes(
+						ImmutableList.of(
+								StorageScopes.DEVSTORAGE_READ_WRITE
+						)
+				)
+				.setServiceAccountPrivateKeyFromP12File(new File(p12KeyFilePath.orNull()))
+				.build();
+	}
+
+	/**
+	 * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
+	 * @see https://developers.google.com/accounts/docs/OAuth2
+	 */
+	private ComputeCredential getComputeCredential() throws IOException
+	{
+		ComputeCredential credential = new ComputeCredential.Builder(httpTransport, jsonFactory)
+				.build();
+		credential.refreshToken();
+
+		return credential;
+	}
+
+	public Storage getGcsClient(String bucket) throws GoogleJsonResponseException, IOException
+	{
+		Storage client = new Storage.Builder(httpTransport, jsonFactory, credentials)
+				.setApplicationName(applicationName)
+				.build();
+
+		// For throw IOException when authentication is fail.
+		long maxResults = 1;
+		Objects objects = client.objects().list(bucket).setMaxResults(maxResults).execute();
+
+		return client;
+	}
+}

data/src/main/java/org/embulk/output/GcsOutputPlugin.java

@@ -1,22 +1,16 @@
 package org.embulk.output;
 
-import com.google.api.client.auth.oauth2.Credential;
-import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
-import com.google.api.client.http.HttpTransport;
 import com.google.api.client.http.InputStreamContent;
-import com.google.api.client.http.apache.ApacheHttpTransport;
-import com.google.api.client.json.JsonFactory;
-import com.google.api.client.json.jackson2.JacksonFactory;
 import com.google.api.services.storage.Storage;
-import com.google.api.services.storage.StorageScopes;
 import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.CommitReport;
+import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigException;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.Buffer;
@@ -25,7 +19,6 @@ import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
 import org.slf4j.Logger;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.PipedInputStream;
 import java.io.PipedOutputStream;
@@ -40,35 +33,40 @@ import java.util.concurrent.Future;
 
 public class GcsOutputPlugin implements FileOutputPlugin {
 	private static final Logger logger = Exec.getLogger(GcsOutputPlugin.class);
-	private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
 
 	public interface PluginTask extends Task {
 		@Config("bucket")
-		public String getBucket();
+		String getBucket();
 
 		@Config("path_prefix")
-		public String getPathPrefix();
+		String getPathPrefix();
 
 		@Config("file_ext")
-		public String getFileNameExtension();
+		String getFileNameExtension();
 
 		@Config("sequence_format")
 		@ConfigDefault("\".%03d.%02d\"")
-		public String getSequenceFormat();
+		String getSequenceFormat();
 
 		@Config("content_type")
 		@ConfigDefault("\"application/octet-stream\"")
-		public String getContentType();
+		String getContentType();
+
+		@Config("auth_method")
+		@ConfigDefault("\"private_key\"")
+		AuthMethod getAuthMethod();
 
 		@Config("service_account_email")
-		public String getServiceAccountEmail();
+		@ConfigDefault("null")
+		Optional<String> getServiceAccountEmail();
 
 		@Config("p12_keyfile_path")
-		public String getP12KeyfilePath();
+		@ConfigDefault("null")
+		Optional<String> getP12KeyfilePath();
 
 		@Config("application_name")
 		@ConfigDefault("\"embulk-output-gcs\"")
-		public String getApplicationName();
+		String getApplicationName();
 	}
 
 	@Override
@@ -90,7 +88,7 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 	@Override
 	public void cleanup(TaskSource taskSource,
 	                    int taskCount,
-	                    List<CommitReport> successCommitReports) {
+	                    List<TaskReport> successTaskReports) {
 	}
 
 	@Override
@@ -101,34 +99,15 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 		return new TransactionalGcsFileOutput(task, client, taskIndex);
 	}
 
-	private GoogleCredential createCredential(final PluginTask task, final HttpTransport httpTransport) {
+	private Storage createClient(final PluginTask task) {
+		Storage client = null;
 		try {
-			// @see https://developers.google.com/accounts/docs/OAuth2ServiceAccount#authorizingrequests
-			// @see https://cloud.google.com/compute/docs/api/how-tos/authorization
-			// @see https://developers.google.com/resources/api-libraries/documentation/storage/v1/java/latest/com/google/api/services/storage/STORAGE_SCOPE.html
-			GoogleCredential cred = new GoogleCredential.Builder()
-					.setTransport(httpTransport)
-					.setJsonFactory(JSON_FACTORY)
-					.setServiceAccountId(task.getServiceAccountEmail())
-					.setServiceAccountScopes(ImmutableList.of(StorageScopes.DEVSTORAGE_READ_WRITE))
-					.setServiceAccountPrivateKeyFromP12File(new File(task.getP12KeyfilePath()))
-					.build();
-			return cred;
-		} catch (IOException ex) {
-			logger.error(String.format("Could not load client secrets file %s", task.getP12KeyfilePath()));
-			throw Throwables.propagate(ex);
-		} catch (GeneralSecurityException ex) {
-			logger.error("Google Authentication was failed");
-			throw Throwables.propagate(ex);
+			GcsAuthentication auth = new GcsAuthentication(task.getAuthMethod().getString(), task.getServiceAccountEmail(), task.getP12KeyfilePath(), task.getApplicationName());
+			client = auth.getGcsClient(task.getBucket());
+		} catch (GeneralSecurityException | IOException ex) {
+			throw new ConfigException(ex);
 		}
-	}
 
-	private Storage createClient(final PluginTask task) {
-		HttpTransport httpTransport = new ApacheHttpTransport.Builder().build();
-		Credential credential = createCredential(task, httpTransport);
-		Storage client = new Storage.Builder(httpTransport, JSON_FACTORY, credential)
-				.setApplicationName(task.getApplicationName())
-				.build();
 		return client;
 	}
 
@@ -194,8 +173,8 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 		}
 
 		@Override
-		public CommitReport commit() {
-			CommitReport report = Exec.newCommitReport();
+		public TaskReport commit() {
+			TaskReport report = Exec.newTaskReport();
 			report.set("files", storageObjects);
 			return report;
 		}
@@ -250,4 +229,22 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 			}
 		}
 	}
+
+	public enum AuthMethod
+	{
+		private_key("private_key"),
+		compute_engine("compute_engine");
+
+		private final String string;
+
+		AuthMethod(String string)
+		{
+			this.string = string;
+		}
+
+		public String getString()
+		{
+			return string;
+		}
+	}
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-gcs
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-15 00:00:00.000000000 Z
+date: 2015-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -59,11 +59,12 @@ files:
 - gradlew
 - gradlew.bat
 - lib/embulk/output/gcs.rb
+- src/main/java/org/embulk/output/GcsAuthentication.java
 - src/main/java/org/embulk/output/GcsOutputPlugin.java
 - src/test/java/org/embulk/output/TestGcsOutputPlugin.java
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-gcs-0.1.1.jar
+- classpath/embulk-output-gcs-0.2.0.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev28-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar