embulk-output-gcs 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 94fccd956be721058fbf74ffd57017d05caf8e5b
-  data.tar.gz: e7e35ef7704225a633d47babdd21c91edd6ad272
+  metadata.gz: 7f2569f6effb1c6cd11617f367c4f9edc5c34955
+  data.tar.gz: a320b195255e974e5bc94872af9674930284bf11
 SHA512:
-  metadata.gz: 2b2325cf0230d16f57fb7a247a9237428fd2d4527cd6848367a5989e3d8bc9a9da7f282b3a81ec97ad2425da9bb70cbd0a26465fcc1dd747ce3e804908cd48ad
-  data.tar.gz: ceb4828ba593c2e2ab8cf043bce26b4509867489cf3b7eac96fd00f1ef10b5ffe8895587a1844d7b01671ea0373cd30b82d39a94da2caf8183cdee206dcdd4e7
+  metadata.gz: d44045b96beeed143a690ce74d93463c35183f32114990a19b48d796f40180e32885014cebda5e2551e514fdbd5c2cf092d16d7ae251d7eb6515a796eaa902d2
+  data.tar.gz: d32896f52c60f511eb5bcdb5c802219d4652e24f8722c2e4f85793f6aab415dc21204b588e9e1fe7e50175b5268d467c9f171e14df37cc3ad30167db67d19363

data/.travis.yml

@@ -0,0 +1,23 @@
+language: java
+jdk:
+- oraclejdk8
+- oraclejdk7
+- openjdk7
+env:
+  global:
+  - GCP_EMAIL=account-2@embulk-output-gcs-test.iam.gserviceaccount.com
+  - GCP_P12_KEYFILE=embulk-output-gcs-test.p12
+  - GCP_JSON_KEYFILE=embulk-output-gcs-test.json
+  - GCP_BUCKET=embulk-output-gcs-test-01
+  - GCP_BUCKET_DIRECTORY=unittests
+before_cache:
+- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
+cache:
+  directories:
+  - "$HOME/.gradle/caches/"
+  - "$HOME/.gradle/wrapper/"
+before_install:
+- openssl aes-256-cbc -K $encrypted_1230b5822723_key -iv $encrypted_1230b5822723_iv
+  -in src/test/resources/keys.tar.enc -out keys.tar -d
+- tar xvf keys.tar
+after_success: "./gradlew jacocoTestReport"

data/README.md

@@ -1,3 +1,5 @@
+[![Build Status](https://travis-ci.org/hakobera/embulk-output-gcs.svg?branch=master)](https://travis-ci.org/hakobera/embulk-output-gcs)
+
 # Google Cloud Storage output plugin for Embulk
 
 Google Cloud Storage output plugin for [Embulk](https://github.com/embulk/embulk).
@@ -15,9 +17,10 @@ Google Cloud Storage output plugin for [Embulk](https://github.com/embulk/embulk
 - **path_prefix**: Prefix of output keys (string, required)
 - **file_ext**: Extention of output file (string, required)
 - **content_type**: content type of output file (string, optional, default value is "application/octet-stream")
-- **auth_method**: Authentication method `private_key` or `compute_engine` (string, optional, default value is "private_key")
-- **service_account_email**: Google Cloud Platform service account email (string, required)
-- **p12_keyfile_path**: Private key file fullpath of Google Cloud Platform service account (string, required)
+- **auth_method**: Authentication method `private_key`, `json_key` or `compute_engine` (string, optional, default value is "private_key")
+- **service_account_email**: Google Cloud Platform service account email (string, required when auth_method is private_key)
+- **p12_keyfile**: Private key file fullpath of Google Cloud Platform service account (string, required when auth_method is private_key)
+- **json_keyfile** fullpath of json_key (string, required when auth_method is json_key)
 - **application_name**: Application name, anything you like (string, optional, default value is "embulk-output-gcs")
 
 ## Example
@@ -30,7 +33,7 @@ out:
   file_ext: .csv
   auth_method: `private_key` #default
   service_account_email: 'XYZ@developer.gserviceaccount.com'
-  p12_keyfile_path: '/path/to/private/key.p12'
+  p12_keyfile: '/path/to/private/key.p12'
   formatter:
     type: csv
     encoding: UTF-8
@@ -38,18 +41,56 @@ out:
 
 ## Authentication
 
-There are two methods supported to fetch access token for the service account.
+There are three methods supported to fetch access token for the service account.
+
+1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
+2. JSON key of GCP(Google Cloud Platform)'s service account
+3. Pre-defined access token (Google Compute Engine only)
+
+### Public-Private key pair of GCP's service account
+
+You first need to create a service account (client ID), download its private key and deploy the key with embulk.
+
+```yaml
+out:
+  type: gcs
+  auth_method: private_key
+  service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
+  p12_keyfile: /path/to/p12_keyfile.p12
+```
 
-1. Public-Private key pair
-2. Pre-defined access token (Compute Engine only)
+### JSON key of GCP's service account
 
-The examples above use the first one.  You first need to create a service account (client ID),
-download its private key and deploy the key with embulk.
+You first need to create a service account (client ID), download its json key and deploy the key with embulk.
+
+```yaml
+out:
+  type: gcs
+  auth_method: json_key
+  json_keyfile: /path/to/json_keyfile.json
+```
+
+You can also embed contents of json_keyfile at config.yml.
+
+```yaml
+out:
+  type: gcs
+  auth_method: json_key
+  json_keyfile:
+    content: |
+      {
+          "private_key_id": "123456789",
+          "private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
+          "client_email": "..."
+       }
+```
+
+### Pre-defined access token(GCE only)
 
 On the other hand, you don't need to explicitly create a service account for embulk when you
-run embulk in Google Compute Engine. In this second authentication method, you need to
+run embulk in Google Compute Engine. In this third authentication method, you need to
 add the API scope "https://www.googleapis.com/auth/devstorage.read_write" to the scope list of your
-Compute Engine instance, then you can configure embulk like this.
+Compute Engine VM instance, then you can configure embulk like this.
 
 [Setting the scope of service account access for instances](https://cloud.google.com/compute/docs/authentication)
 
@@ -64,3 +105,53 @@ out:
 ```
 $ ./gradlew gem
 ```
+
+## Test
+
+```
+$ ./gradlew test  # -t to watch change of files and rebuild continuously
+```
+
+To run unit tests, we need to configure the following environment variables.
+
+When environment variables are not set, skip almost test cases.
+
+```
+GCP_EMAIL
+GCP_P12_KEYFILE
+GCP_JSON_KEYFILE
+GCP_BUCKET
+GCP_BUCKET_DIRECTORY(optional, if needed)
+```
+
+If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
+```
+$ vi ~/Library/LaunchAgents/environment.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>my.startup</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>sh</string>
+    <string>-c</string>
+    <string>
+      launchctl setenv GCP_EMAIL ABCXYZ123ABCXYZ123.gserviceaccount.com
+      launchctl setenv GCP_P12_KEYFILE /path/to/p12_keyfile.p12
+      launchctl setenv GCP_JSON_KEYFILE /path/to/json_keyfile.json
+      launchctl setenv GCP_BUCKET my-bucket
+      launchctl setenv GCP_BUCKET_DIRECTORY unittests
+    </string>
+  </array>
+  <key>RunAtLoad</key>
+  <true/>
+</dict>
+</plist>
+
+$ launchctl load ~/Library/LaunchAgents/environment.plist
+$ launchctl getenv GCP_EMAIL //try to get value.
+
+Then start your applications.
+```

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -15,7 +16,7 @@ configurations {
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
-version = "0.2.0"
+version = "0.3.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.3"
@@ -24,7 +25,9 @@ dependencies {
     compile "com.google.http-client:google-http-client-jackson2:1.19.0"
     compile ("com.google.apis:google-api-services-storage:v1-rev28-1.19.1") {exclude module: "guava-jdk5"}
 
-    testCompile "junit:junit:4.+"
+    testCompile "junit:junit:4.12"
+    testCompile "org.embulk:embulk-core:0.7.5:tests"
+    testCompile "org.embulk:embulk-standards:0.7.5"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
@@ -60,3 +63,7 @@ Gem::Specification.new do |spec|
 end
 /$)
 }
+
+task gempush << {
+  "gem push pkg/embulk-output-gcs-${project.version}.gem".execute().waitFor()
+}

data/src/main/java/org/embulk/output/GcsAuthentication.java

@@ -1,8 +1,9 @@
 package org.embulk.output;
 
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
-
+import java.util.Collections;
 import com.google.api.client.http.apache.ApacheHttpTransport;
 import com.google.api.services.storage.model.Objects;
 import com.google.common.base.Optional;
@@ -25,16 +26,19 @@ public class GcsAuthentication
 	private final Logger log = Exec.getLogger(GcsAuthentication.class);
 	private final Optional<String> serviceAccountEmail;
 	private final Optional<String> p12KeyFilePath;
+	private final Optional<String> jsonKeyFilePath;
 	private final String applicationName;
 	private final HttpTransport httpTransport;
 	private final JsonFactory jsonFactory;
 	private final HttpRequestInitializer credentials;
 
-	public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail, Optional<String> p12KeyFilePath, String applicationName)
-			throws IOException, GeneralSecurityException
+	public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
+					Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
+					throws IOException, GeneralSecurityException
 	{
 		this.serviceAccountEmail = serviceAccountEmail;
 		this.p12KeyFilePath = p12KeyFilePath;
+		this.jsonKeyFilePath = jsonKeyFilePath;
 		this.applicationName = applicationName;
 
 		this.httpTransport = new ApacheHttpTransport.Builder().build();
@@ -42,6 +46,8 @@ public class GcsAuthentication
 
 		if (authMethod.equals("compute_engine")) {
 			this.credentials = getComputeCredential();
+		} else if(authMethod.toLowerCase().equals("json_key")) {
+			this.credentials = getServiceAccountCredentialFromJsonFile();
 		} else {
 			this.credentials = getServiceAccountCredential();
 		}
@@ -68,6 +74,14 @@ public class GcsAuthentication
 				.build();
 	}
 
+	private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
+	{
+		FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
+
+		return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
+				.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE));
+	}
+
 	/**
 	 * @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
 	 * @see https://developers.google.com/accounts/docs/OAuth2

data/src/main/java/org/embulk/output/GcsOutputPlugin.java

@@ -3,6 +3,7 @@ package org.embulk.output;
 import com.google.api.client.http.InputStreamContent;
 import com.google.api.services.storage.Storage;
 import com.google.api.services.storage.model.StorageObject;
+import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.embulk.config.TaskReport;
@@ -17,6 +18,7 @@ import org.embulk.spi.Buffer;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutputPlugin;
 import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.unit.LocalFile;
 import org.slf4j.Logger;
 
 import java.io.IOException;
@@ -60,10 +62,20 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 		@ConfigDefault("null")
 		Optional<String> getServiceAccountEmail();
 
+		// kept for backward compatibility
 		@Config("p12_keyfile_path")
 		@ConfigDefault("null")
 		Optional<String> getP12KeyfilePath();
 
+		@Config("p12_keyfile")
+		@ConfigDefault("null")
+		Optional<LocalFile> getP12Keyfile();
+		void setP12Keyfile(Optional<LocalFile> p12Keyfile);
+
+		@Config("json_keyfile")
+		@ConfigDefault("null")
+		Optional<LocalFile> getJsonKeyfile();
+
 		@Config("application_name")
 		@ConfigDefault("\"embulk-output-gcs\"")
 		String getApplicationName();
@@ -74,6 +86,28 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 	                              int taskCount,
 	                              FileOutputPlugin.Control control) {
 		PluginTask task = config.loadConfig(PluginTask.class);
+
+		if (task.getP12KeyfilePath().isPresent()) {
+			if (task.getP12Keyfile().isPresent()) {
+				throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
+			}
+			try {
+				task.setP12Keyfile(Optional.of(LocalFile.of(task.getP12KeyfilePath().get())));
+			} catch (IOException ex) {
+				throw Throwables.propagate(ex);
+			}
+		}
+
+		if (task.getAuthMethod().getString().equals("json_key")) {
+			if (!task.getJsonKeyfile().isPresent()) {
+				throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
+			}
+		} else if (task.getAuthMethod().getString().equals("private_key")) {
+			if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
+				throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
+			}
+		}
+
 		return resume(task.dump(), taskCount, control);
 	}
 
@@ -99,18 +133,43 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 		return new TransactionalGcsFileOutput(task, client, taskIndex);
 	}
 
+	private GcsAuthentication newGcsAuth(PluginTask task)
+	{
+		try {
+			return new GcsAuthentication(
+					task.getAuthMethod().getString(),
+					task.getServiceAccountEmail(),
+					task.getP12Keyfile().transform(localFileToPathString()),
+					task.getJsonKeyfile().transform(localFileToPathString()),
+					task.getApplicationName()
+			);
+		} catch (GeneralSecurityException | IOException ex) {
+			throw new ConfigException(ex);
+		}
+	}
+
 	private Storage createClient(final PluginTask task) {
 		Storage client = null;
 		try {
-			GcsAuthentication auth = new GcsAuthentication(task.getAuthMethod().getString(), task.getServiceAccountEmail(), task.getP12KeyfilePath(), task.getApplicationName());
+			GcsAuthentication auth = newGcsAuth(task);
 			client = auth.getGcsClient(task.getBucket());
-		} catch (GeneralSecurityException | IOException ex) {
+		} catch (ConfigException | IOException ex) {
 			throw new ConfigException(ex);
 		}
 
 		return client;
 	}
 
+	private Function<LocalFile, String> localFileToPathString() {
+		return new Function<LocalFile, String>()
+		{
+			public String apply(LocalFile file)
+			{
+				return file.getPath().toString();
+			}
+		};
+	}
+
 	static class TransactionalGcsFileOutput implements TransactionalFileOutput {
 		private final int taskIndex;
 		private final Storage client;
@@ -233,7 +292,8 @@ public class GcsOutputPlugin implements FileOutputPlugin {
 	public enum AuthMethod
 	{
 		private_key("private_key"),
-		compute_engine("compute_engine");
+		compute_engine("compute_engine"),
+		json_key("json_key");
 
 		private final String string;
 

data/src/test/java/org/embulk/output/TestGcsAuthentication.java

@@ -0,0 +1,177 @@
+package org.embulk.output;
+
+import com.google.common.base.Optional;
+import org.embulk.EmbulkTestRuntime;
+import com.google.api.services.storage.Storage;
+import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
+
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.security.GeneralSecurityException;
+import com.google.api.client.googleapis.json.GoogleJsonResponseException;
+
+import java.lang.reflect.Field;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+
+public class TestGcsAuthentication
+{
+	private static Optional<String> GCP_EMAIL;
+	private static Optional<String> GCP_P12_KEYFILE;
+	private static Optional<String> GCP_JSON_KEYFILE;
+	private static String GCP_BUCKET;
+	private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
+
+	/*
+	 * This test case requires environment variables
+	 *   GCP_EMAIL
+	 *   GCP_P12_KEYFILE
+	 *   GCP_JSON_KEYFILE
+	 *   GCP_BUCKET
+	 */
+	@BeforeClass
+	public static void initializeConstant()
+	{
+		GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
+		GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
+		GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+		GCP_BUCKET = System.getenv("GCP_BUCKET");
+		// skip test cases, if environment variables are not set.
+		assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
+	}
+
+	@Rule
+	public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+
+	@Test
+	public void testGetServiceAccountCredentialSuccess()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"private_key",
+				GCP_EMAIL,
+				GCP_P12_KEYFILE,
+				null,
+				GCP_APPLICATION_NAME
+		);
+
+		Field field = GcsAuthentication.class.getDeclaredField("credentials");
+		field.setAccessible(true);
+
+		assertEquals(GoogleCredential.class, field.get(auth).getClass());
+	}
+
+	@Test(expected = FileNotFoundException.class)
+	public void testGetServiceAccountCredentialThrowFileNotFoundException()
+			throws GeneralSecurityException, IOException
+	{
+		Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
+		GcsAuthentication auth = new GcsAuthentication(
+				"private_key",
+				GCP_EMAIL,
+				notFoundP12Keyfile,
+				null,
+				GCP_APPLICATION_NAME
+		);
+	}
+
+	@Test
+	public void testGetGcsClientUsingServiceAccountCredentialSuccess()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"private_key",
+				GCP_EMAIL,
+				GCP_P12_KEYFILE,
+				null,
+				GCP_APPLICATION_NAME
+		);
+
+		Storage client = auth.getGcsClient(GCP_BUCKET);
+
+		assertEquals(Storage.class, client.getClass());
+	}
+
+	@Test(expected = GoogleJsonResponseException.class)
+	public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"private_key",
+				GCP_EMAIL,
+				GCP_P12_KEYFILE,
+				null,
+				GCP_APPLICATION_NAME
+		);
+
+		Storage client = auth.getGcsClient("non-exists-bucket");
+
+		assertEquals(Storage.class, client.getClass());
+	}
+
+	@Test
+	public void testGetServiceAccountCredentialFromJsonFileSuccess()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"json_key",
+				GCP_EMAIL,
+				null,
+				GCP_JSON_KEYFILE,
+				GCP_APPLICATION_NAME
+		);
+		Field field = GcsAuthentication.class.getDeclaredField("credentials");
+		field.setAccessible(true);
+
+		assertEquals(GoogleCredential.class, field.get(auth).getClass());
+	}
+
+	@Test(expected = FileNotFoundException.class)
+	public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
+			throws GeneralSecurityException, IOException
+	{
+		Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
+		GcsAuthentication auth = new GcsAuthentication(
+				"json_key",
+				GCP_EMAIL,
+				null,
+				notFoundJsonKeyfile,
+				GCP_APPLICATION_NAME
+		);
+	}
+
+	@Test
+	public void testGetServiceAccountCredentialFromJsonSuccess()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"json_key",
+				GCP_EMAIL,
+				null,
+				GCP_JSON_KEYFILE,
+				GCP_APPLICATION_NAME
+		);
+
+		Storage client = auth.getGcsClient(GCP_BUCKET);
+
+		assertEquals(Storage.class, client.getClass());
+	}
+
+	@Test(expected = GoogleJsonResponseException.class)
+	public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
+			throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
+	{
+		GcsAuthentication auth = new GcsAuthentication(
+				"json_key",
+				GCP_EMAIL,
+				null,
+				GCP_JSON_KEYFILE,
+				GCP_APPLICATION_NAME
+		);
+
+		Storage client = auth.getGcsClient("non-exists-bucket");
+	}
+}

data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java

@@ -1,5 +1,426 @@
 package org.embulk.output;
 
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.base.Optional;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Buffer;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileOutputPlugin;
+import org.embulk.spi.FileOutputRunner;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.TransactionalFileOutput;
+import org.embulk.spi.Schema;
+import org.embulk.output.GcsOutputPlugin.PluginTask;
+import org.embulk.standards.CsvParserPlugin;
+
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assume.assumeNotNull;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
+
+import com.google.api.services.storage.Storage;
+
 public class TestGcsOutputPlugin
 {
+	private static Optional<String> GCP_EMAIL;
+	private static Optional<String> GCP_P12_KEYFILE;
+	private static Optional<String> GCP_JSON_KEYFILE;
+	private static String GCP_BUCKET;
+	private static String GCP_BUCKET_DIRECTORY;
+	private static String GCP_PATH_PREFIX;
+	private static String LOCAL_PATH_PREFIX;
+	private final String GCP_APPLICATION_NAME = "embulk-output-gcs";
+	private FileOutputRunner runner;
+
+	/*
+	 * This test case requires environment variables
+	 *   GCP_EMAIL
+	 *   GCP_P12_KEYFILE
+	 *   GCP_JSON_KEYFILE
+	 *   GCP_BUCKET
+	 */
+	@BeforeClass
+	public static void initializeConstant()
+	{
+		GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
+		GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
+		GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
+		GCP_BUCKET = System.getenv("GCP_BUCKET");
+		// skip test cases, if environment variables are not set.
+		assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
+
+		GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
+		GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
+		LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
+	}
+
+	@Rule
+	public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+	private GcsOutputPlugin plugin;
+
+	@Before
+	public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
+	{
+		plugin = new GcsOutputPlugin();
+		runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
+	}
+
+	@Test
+	public void checkDefaultValues()
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("formatter", formatterConfig());
+
+		GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
+		assertEquals("private_key", task.getAuthMethod().toString());
+	}
+
+	// p12_keyfile is null when auth_method is private_key
+	@Test(expected = ConfigException.class)
+	public void checkDefaultValuesP12keyNull()
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("auth_method", "private_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("p12_keyfile", null)
+				.set("formatter", formatterConfig());
+
+		Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+		runner.transaction(config, schema, 0, new Control());
+	}
+
+	// both p12_keyfile and p12_keyfile_path set
+	@Test(expected = ConfigException.class)
+	public void checkDefaultValuesConflictSetting()
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("auth_method", "private_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("p12_keyfile", GCP_P12_KEYFILE)
+				.set("p12_keyfile_path", GCP_P12_KEYFILE)
+				.set("formatter", formatterConfig());
+
+		Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+		runner.transaction(config, schema, 0, new Control());
+	}
+
+	// invalid p12keyfile when auth_method is private_key
+	@Test(expected = ConfigException.class)
+	public void checkDefaultValuesInvalidPrivateKey()
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("auth_method", "private_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("p12_keyfile", "invalid-key.p12")
+				.set("formatter", formatterConfig());
+
+		Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+		runner.transaction(config, schema, 0, new Control());
+	}
+
+	// json_keyfile is null when auth_method is json_key
+	@Test(expected = ConfigException.class)
+	public void checkDefaultValuesJsonKeyfileNull()
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("auth_method", "json_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("json_keyfile", null)
+				.set("formatter", formatterConfig());
+
+		Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+		runner.transaction(config, schema, 0, new Control());
+	}
+
+	@Test
+	public void testGcsClientCreateSuccessfully()
+			throws GeneralSecurityException, IOException, NoSuchMethodException,
+			IllegalAccessException, InvocationTargetException
+	{
+		ConfigSource configSource = config();
+		PluginTask task = configSource.loadConfig(PluginTask.class);
+		Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+		runner.transaction(configSource, schema, 0, new Control());
+
+		Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
+		method.setAccessible(true);
+		method.invoke(plugin, task); // no errors happens
+	}
+
+	@Test(expected = ConfigException.class)
+	public void testGcsClientCreateThrowConfigException()
+			throws GeneralSecurityException, IOException, NoSuchMethodException,
+			IllegalAccessException, InvocationTargetException
+	{
+		ConfigSource config = Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", "non-exists-bucket")
+				.set("path_prefix", "my-prefix")
+				.set("file_ext", ".csv")
+				.set("auth_method", "json_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("json_keyfile", GCP_JSON_KEYFILE)
+				.set("formatter", formatterConfig());
+
+		PluginTask task = config.loadConfig(PluginTask.class);
+
+		Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+		runner.transaction(config, schema, 0, new Control());
+
+		Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
+		method.setAccessible(true);
+		try {
+			method.invoke(plugin, task);
+		} catch (InvocationTargetException ex) {
+			throw (ConfigException) ex.getCause();
+		}
+	}
+
+	@Test
+	public void testResume()
+	{
+		PluginTask task = config().loadConfig(PluginTask.class);
+		plugin.resume(task.dump(), 0, new FileOutputPlugin.Control()  // no errors happens
+		{
+			@Override
+			public List<TaskReport> run(TaskSource taskSource)
+			{
+				return Lists.newArrayList(Exec.newTaskReport());
+			}
+		});
+	}
+
+	@Test
+	public void testCleanup()
+	{
+		PluginTask task = config().loadConfig(PluginTask.class);
+		plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+	}
+
+	@Test
+	public void testGcsFileOutputByOpen() throws Exception
+	{
+		ConfigSource configSource = config();
+		PluginTask task = configSource.loadConfig(PluginTask.class);
+		Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+		runner.transaction(configSource, schema, 0, new Control());
+
+		TransactionalFileOutput output = plugin.open(task.dump(), 0);
+
+		output.nextFile();
+
+		FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
+		byte[] bytes = convertInputStreamToByte(is);
+		Buffer buffer = Buffer.wrap(bytes);
+		output.add(buffer);
+
+		output.finish();
+		output.commit();
+
+		String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
+		assertRecords(remotePath);
+	}
+
+	public ConfigSource config()
+	{
+		return Exec.newConfigSource()
+				.set("in", inputConfig())
+				.set("parser", parserConfig(schemaConfig()))
+				.set("type", "gcs")
+				.set("bucket", GCP_BUCKET)
+				.set("path_prefix", GCP_PATH_PREFIX)
+				.set("last_path", "")
+				.set("file_ext", ".csv")
+				.set("auth_method", "private_key")
+				.set("service_account_email", GCP_EMAIL)
+				.set("p12_keyfile", GCP_P12_KEYFILE)
+				.set("json_keyfile", GCP_JSON_KEYFILE)
+				.set("application_name", GCP_APPLICATION_NAME)
+				.set("formatter", formatterConfig());
+	}
+
+	private class Control
+			implements OutputPlugin.Control
+	{
+		@Override
+		public List<TaskReport> run(TaskSource taskSource)
+		{
+			return Lists.newArrayList(Exec.newTaskReport());
+		}
+	}
+
+	private ImmutableMap<String, Object> inputConfig()
+	{
+		ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+		builder.put("type", "file");
+		builder.put("path_prefix", LOCAL_PATH_PREFIX);
+		builder.put("last_path", "");
+		return builder.build();
+	}
+
+	private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+	{
+		ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+		builder.put("type", "csv");
+		builder.put("newline", "CRLF");
+		builder.put("delimiter", ",");
+		builder.put("quote", "\"");
+		builder.put("escape", "\"");
+		builder.put("trim_if_not_quoted", false);
+		builder.put("skip_header_lines", 1);
+		builder.put("allow_extra_columns", false);
+		builder.put("allow_optional_columns", false);
+		builder.put("columns", schemaConfig);
+		return builder.build();
+	}
+
+	private ImmutableList<Object> schemaConfig()
+	{
+		ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+		builder.add(ImmutableMap.of("name", "id", "type", "long"));
+		builder.add(ImmutableMap.of("name", "account", "type", "long"));
+		builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+		builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+		builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+		return builder.build();
+	}
+
+	private ImmutableMap<String, Object> formatterConfig()
+	{
+		ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+		builder.put("type", "csv");
+		builder.put("header_line", "false");
+		builder.put("timezone", "Asia/Tokyo");
+		return builder.build();
+	}
+
+	private void assertRecords(String gcsPath) throws Exception
+	{
+		ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
+		assertEquals(5, records.size());
+		{
+			List<String> record = records.get(1);
+			assertEquals("1", record.get(0));
+			assertEquals("32864", record.get(1));
+			assertEquals("2015-01-27 19:23:49", record.get(2));
+			assertEquals("20150127", record.get(3));
+			assertEquals("embulk", record.get(4));
+		}
+
+		{
+			List<String> record = records.get(2);
+			assertEquals("2", record.get(0));
+			assertEquals("14824", record.get(1));
+			assertEquals("2015-01-27 19:01:23", record.get(2));
+			assertEquals("20150127", record.get(3));
+			assertEquals("embulk jruby", record.get(4));
+		}
+	}
+
+	private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
+	{
+		ConfigSource config = config();
+
+		PluginTask task = config.loadConfig(PluginTask.class);
+
+		Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
+		method.setAccessible(true);
+		Storage client = (Storage) method.invoke(plugin, task);
+		Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
+
+		ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
+
+		InputStream is =  getObject.executeMediaAsInputStream();
+		BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+		String line;
+		while ((line = reader.readLine()) != null) {
+			List<String> records = Arrays.asList(line.split(",", 0));
+
+			builder.add(records);
+		}
+		return builder.build();
+	}
+
+	private static String getDirectory(String dir)
+	{
+		if (dir != null && !dir.endsWith("/")) {
+			dir = dir + "/";
+		}
+		if (dir.startsWith("/")) {
+			dir = dir.replaceFirst("/", "");
+		}
+		return dir;
+	}
+
+	private byte[] convertInputStreamToByte(InputStream is) throws IOException
+	{
+		ByteArrayOutputStream bo = new ByteArrayOutputStream();
+		byte [] buffer = new byte[1024];
+		while(true) {
+			int len = is.read(buffer);
+			if(len < 0) {
+				break;
+			}
+			bo.write(buffer, 0, len);
+		}
+		return bo.toByteArray();
+	}
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/src/test/resources/sample_02.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-gcs
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Kazuyuki Honda
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-04 00:00:00.000000000 Z
+date: 2015-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - LICENSE.txt
 - README.md
 - build.gradle
@@ -61,10 +62,14 @@ files:
 - lib/embulk/output/gcs.rb
 - src/main/java/org/embulk/output/GcsAuthentication.java
 - src/main/java/org/embulk/output/GcsOutputPlugin.java
+- src/test/java/org/embulk/output/TestGcsAuthentication.java
 - src/test/java/org/embulk/output/TestGcsOutputPlugin.java
+- src/test/resources/keys.tar.enc
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/commons-codec-1.3.jar
 - classpath/commons-logging-1.1.1.jar
-- classpath/embulk-output-gcs-0.2.0.jar
+- classpath/embulk-output-gcs-0.3.0.jar
 - classpath/google-api-client-1.19.1.jar
 - classpath/google-api-services-storage-v1-rev28-1.19.1.jar
 - classpath/google-http-client-1.19.0.jar