embulk-output-gcs 0.2.0 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +23 -0
- data/README.md +102 -11
- data/build.gradle +9 -2
- data/classpath/embulk-output-gcs-0.3.0.jar +0 -0
- data/src/main/java/org/embulk/output/GcsAuthentication.java +17 -3
- data/src/main/java/org/embulk/output/GcsOutputPlugin.java +63 -3
- data/src/test/java/org/embulk/output/TestGcsAuthentication.java +177 -0
- data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java +421 -0
- data/src/test/resources/keys.tar.enc +0 -0
- data/src/test/resources/sample_01.csv +5 -0
- data/src/test/resources/sample_02.csv +5 -0
- metadata +8 -3
- data/classpath/embulk-output-gcs-0.2.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7f2569f6effb1c6cd11617f367c4f9edc5c34955
|
4
|
+
data.tar.gz: a320b195255e974e5bc94872af9674930284bf11
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d44045b96beeed143a690ce74d93463c35183f32114990a19b48d796f40180e32885014cebda5e2551e514fdbd5c2cf092d16d7ae251d7eb6515a796eaa902d2
|
7
|
+
data.tar.gz: d32896f52c60f511eb5bcdb5c802219d4652e24f8722c2e4f85793f6aab415dc21204b588e9e1fe7e50175b5268d467c9f171e14df37cc3ad30167db67d19363
|
data/.travis.yml
ADDED
@@ -0,0 +1,23 @@
|
|
1
|
+
language: java
|
2
|
+
jdk:
|
3
|
+
- oraclejdk8
|
4
|
+
- oraclejdk7
|
5
|
+
- openjdk7
|
6
|
+
env:
|
7
|
+
global:
|
8
|
+
- GCP_EMAIL=account-2@embulk-output-gcs-test.iam.gserviceaccount.com
|
9
|
+
- GCP_P12_KEYFILE=embulk-output-gcs-test.p12
|
10
|
+
- GCP_JSON_KEYFILE=embulk-output-gcs-test.json
|
11
|
+
- GCP_BUCKET=embulk-output-gcs-test-01
|
12
|
+
- GCP_BUCKET_DIRECTORY=unittests
|
13
|
+
before_cache:
|
14
|
+
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
|
15
|
+
cache:
|
16
|
+
directories:
|
17
|
+
- "$HOME/.gradle/caches/"
|
18
|
+
- "$HOME/.gradle/wrapper/"
|
19
|
+
before_install:
|
20
|
+
- openssl aes-256-cbc -K $encrypted_1230b5822723_key -iv $encrypted_1230b5822723_iv
|
21
|
+
-in src/test/resources/keys.tar.enc -out keys.tar -d
|
22
|
+
- tar xvf keys.tar
|
23
|
+
after_success: "./gradlew jacocoTestReport"
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/hakobera/embulk-output-gcs.svg?branch=master)](https://travis-ci.org/hakobera/embulk-output-gcs)
|
2
|
+
|
1
3
|
# Google Cloud Storage output plugin for Embulk
|
2
4
|
|
3
5
|
Google Cloud Storage output plugin for [Embulk](https://github.com/embulk/embulk).
|
@@ -15,9 +17,10 @@ Google Cloud Storage output plugin for [Embulk](https://github.com/embulk/embulk
|
|
15
17
|
- **path_prefix**: Prefix of output keys (string, required)
|
16
18
|
- **file_ext**: Extention of output file (string, required)
|
17
19
|
- **content_type**: content type of output file (string, optional, default value is "application/octet-stream")
|
18
|
-
- **auth_method**: Authentication method `private_key` or `compute_engine` (string, optional, default value is "private_key")
|
19
|
-
- **service_account_email**: Google Cloud Platform service account email (string, required)
|
20
|
-
- **
|
20
|
+
- **auth_method**: Authentication method `private_key`, `json_key` or `compute_engine` (string, optional, default value is "private_key")
|
21
|
+
- **service_account_email**: Google Cloud Platform service account email (string, required when auth_method is private_key)
|
22
|
+
- **p12_keyfile**: Private key file fullpath of Google Cloud Platform service account (string, required when auth_method is private_key)
|
23
|
+
- **json_keyfile** fullpath of json_key (string, required when auth_method is json_key)
|
21
24
|
- **application_name**: Application name, anything you like (string, optional, default value is "embulk-output-gcs")
|
22
25
|
|
23
26
|
## Example
|
@@ -30,7 +33,7 @@ out:
|
|
30
33
|
file_ext: .csv
|
31
34
|
auth_method: `private_key` #default
|
32
35
|
service_account_email: 'XYZ@developer.gserviceaccount.com'
|
33
|
-
|
36
|
+
p12_keyfile: '/path/to/private/key.p12'
|
34
37
|
formatter:
|
35
38
|
type: csv
|
36
39
|
encoding: UTF-8
|
@@ -38,18 +41,56 @@ out:
|
|
38
41
|
|
39
42
|
## Authentication
|
40
43
|
|
41
|
-
There are
|
44
|
+
There are three methods supported to fetch access token for the service account.
|
45
|
+
|
46
|
+
1. Public-Private key pair of GCP(Google Cloud Platform)'s service account
|
47
|
+
2. JSON key of GCP(Google Cloud Platform)'s service account
|
48
|
+
3. Pre-defined access token (Google Compute Engine only)
|
49
|
+
|
50
|
+
### Public-Private key pair of GCP's service account
|
51
|
+
|
52
|
+
You first need to create a service account (client ID), download its private key and deploy the key with embulk.
|
53
|
+
|
54
|
+
```yaml
|
55
|
+
out:
|
56
|
+
type: gcs
|
57
|
+
auth_method: private_key
|
58
|
+
service_account_email: ABCXYZ123ABCXYZ123.gserviceaccount.com
|
59
|
+
p12_keyfile: /path/to/p12_keyfile.p12
|
60
|
+
```
|
42
61
|
|
43
|
-
|
44
|
-
2. Pre-defined access token (Compute Engine only)
|
62
|
+
### JSON key of GCP's service account
|
45
63
|
|
46
|
-
|
47
|
-
|
64
|
+
You first need to create a service account (client ID), download its json key and deploy the key with embulk.
|
65
|
+
|
66
|
+
```yaml
|
67
|
+
out:
|
68
|
+
type: gcs
|
69
|
+
auth_method: json_key
|
70
|
+
json_keyfile: /path/to/json_keyfile.json
|
71
|
+
```
|
72
|
+
|
73
|
+
You can also embed contents of json_keyfile at config.yml.
|
74
|
+
|
75
|
+
```yaml
|
76
|
+
out:
|
77
|
+
type: gcs
|
78
|
+
auth_method: json_key
|
79
|
+
json_keyfile:
|
80
|
+
content: |
|
81
|
+
{
|
82
|
+
"private_key_id": "123456789",
|
83
|
+
"private_key": "-----BEGIN PRIVATE KEY-----\nABCDEF",
|
84
|
+
"client_email": "..."
|
85
|
+
}
|
86
|
+
```
|
87
|
+
|
88
|
+
### Pre-defined access token(GCE only)
|
48
89
|
|
49
90
|
On the other hand, you don't need to explicitly create a service account for embulk when you
|
50
|
-
run embulk in Google Compute Engine. In this
|
91
|
+
run embulk in Google Compute Engine. In this third authentication method, you need to
|
51
92
|
add the API scope "https://www.googleapis.com/auth/devstorage.read_write" to the scope list of your
|
52
|
-
Compute Engine instance, then you can configure embulk like this.
|
93
|
+
Compute Engine VM instance, then you can configure embulk like this.
|
53
94
|
|
54
95
|
[Setting the scope of service account access for instances](https://cloud.google.com/compute/docs/authentication)
|
55
96
|
|
@@ -64,3 +105,53 @@ out:
|
|
64
105
|
```
|
65
106
|
$ ./gradlew gem
|
66
107
|
```
|
108
|
+
|
109
|
+
## Test
|
110
|
+
|
111
|
+
```
|
112
|
+
$ ./gradlew test # -t to watch change of files and rebuild continuously
|
113
|
+
```
|
114
|
+
|
115
|
+
To run unit tests, we need to configure the following environment variables.
|
116
|
+
|
117
|
+
When environment variables are not set, skip almost test cases.
|
118
|
+
|
119
|
+
```
|
120
|
+
GCP_EMAIL
|
121
|
+
GCP_P12_KEYFILE
|
122
|
+
GCP_JSON_KEYFILE
|
123
|
+
GCP_BUCKET
|
124
|
+
GCP_BUCKET_DIRECTORY(optional, if needed)
|
125
|
+
```
|
126
|
+
|
127
|
+
If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
|
128
|
+
```
|
129
|
+
$ vi ~/Library/LaunchAgents/environment.plist
|
130
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
131
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
132
|
+
<plist version="1.0">
|
133
|
+
<dict>
|
134
|
+
<key>Label</key>
|
135
|
+
<string>my.startup</string>
|
136
|
+
<key>ProgramArguments</key>
|
137
|
+
<array>
|
138
|
+
<string>sh</string>
|
139
|
+
<string>-c</string>
|
140
|
+
<string>
|
141
|
+
launchctl setenv GCP_EMAIL ABCXYZ123ABCXYZ123.gserviceaccount.com
|
142
|
+
launchctl setenv GCP_P12_KEYFILE /path/to/p12_keyfile.p12
|
143
|
+
launchctl setenv GCP_JSON_KEYFILE /path/to/json_keyfile.json
|
144
|
+
launchctl setenv GCP_BUCKET my-bucket
|
145
|
+
launchctl setenv GCP_BUCKET_DIRECTORY unittests
|
146
|
+
</string>
|
147
|
+
</array>
|
148
|
+
<key>RunAtLoad</key>
|
149
|
+
<true/>
|
150
|
+
</dict>
|
151
|
+
</plist>
|
152
|
+
|
153
|
+
$ launchctl load ~/Library/LaunchAgents/environment.plist
|
154
|
+
$ launchctl getenv GCP_EMAIL //try to get value.
|
155
|
+
|
156
|
+
Then start your applications.
|
157
|
+
```
|
data/build.gradle
CHANGED
@@ -2,6 +2,7 @@ plugins {
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
4
|
id "java"
|
5
|
+
id "jacoco"
|
5
6
|
}
|
6
7
|
import com.github.jrubygradle.JRubyExec
|
7
8
|
repositories {
|
@@ -15,7 +16,7 @@ configurations {
|
|
15
16
|
sourceCompatibility = 1.7
|
16
17
|
targetCompatibility = 1.7
|
17
18
|
|
18
|
-
version = "0.
|
19
|
+
version = "0.3.0"
|
19
20
|
|
20
21
|
dependencies {
|
21
22
|
compile "org.embulk:embulk-core:0.7.3"
|
@@ -24,7 +25,9 @@ dependencies {
|
|
24
25
|
compile "com.google.http-client:google-http-client-jackson2:1.19.0"
|
25
26
|
compile ("com.google.apis:google-api-services-storage:v1-rev28-1.19.1") {exclude module: "guava-jdk5"}
|
26
27
|
|
27
|
-
testCompile "junit:junit:4
|
28
|
+
testCompile "junit:junit:4.12"
|
29
|
+
testCompile "org.embulk:embulk-core:0.7.5:tests"
|
30
|
+
testCompile "org.embulk:embulk-standards:0.7.5"
|
28
31
|
}
|
29
32
|
|
30
33
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
@@ -60,3 +63,7 @@ Gem::Specification.new do |spec|
|
|
60
63
|
end
|
61
64
|
/$)
|
62
65
|
}
|
66
|
+
|
67
|
+
task gempush << {
|
68
|
+
"gem push pkg/embulk-output-gcs-${project.version}.gem".execute().waitFor()
|
69
|
+
}
|
Binary file
|
@@ -1,8 +1,9 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
3
|
import java.io.File;
|
4
|
+
import java.io.FileInputStream;
|
4
5
|
import java.io.IOException;
|
5
|
-
|
6
|
+
import java.util.Collections;
|
6
7
|
import com.google.api.client.http.apache.ApacheHttpTransport;
|
7
8
|
import com.google.api.services.storage.model.Objects;
|
8
9
|
import com.google.common.base.Optional;
|
@@ -25,16 +26,19 @@ public class GcsAuthentication
|
|
25
26
|
private final Logger log = Exec.getLogger(GcsAuthentication.class);
|
26
27
|
private final Optional<String> serviceAccountEmail;
|
27
28
|
private final Optional<String> p12KeyFilePath;
|
29
|
+
private final Optional<String> jsonKeyFilePath;
|
28
30
|
private final String applicationName;
|
29
31
|
private final HttpTransport httpTransport;
|
30
32
|
private final JsonFactory jsonFactory;
|
31
33
|
private final HttpRequestInitializer credentials;
|
32
34
|
|
33
|
-
public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
34
|
-
|
35
|
+
public GcsAuthentication(String authMethod, Optional<String> serviceAccountEmail,
|
36
|
+
Optional<String> p12KeyFilePath, Optional<String> jsonKeyFilePath, String applicationName)
|
37
|
+
throws IOException, GeneralSecurityException
|
35
38
|
{
|
36
39
|
this.serviceAccountEmail = serviceAccountEmail;
|
37
40
|
this.p12KeyFilePath = p12KeyFilePath;
|
41
|
+
this.jsonKeyFilePath = jsonKeyFilePath;
|
38
42
|
this.applicationName = applicationName;
|
39
43
|
|
40
44
|
this.httpTransport = new ApacheHttpTransport.Builder().build();
|
@@ -42,6 +46,8 @@ public class GcsAuthentication
|
|
42
46
|
|
43
47
|
if (authMethod.equals("compute_engine")) {
|
44
48
|
this.credentials = getComputeCredential();
|
49
|
+
} else if(authMethod.toLowerCase().equals("json_key")) {
|
50
|
+
this.credentials = getServiceAccountCredentialFromJsonFile();
|
45
51
|
} else {
|
46
52
|
this.credentials = getServiceAccountCredential();
|
47
53
|
}
|
@@ -68,6 +74,14 @@ public class GcsAuthentication
|
|
68
74
|
.build();
|
69
75
|
}
|
70
76
|
|
77
|
+
private GoogleCredential getServiceAccountCredentialFromJsonFile() throws IOException
|
78
|
+
{
|
79
|
+
FileInputStream stream = new FileInputStream(jsonKeyFilePath.orNull());
|
80
|
+
|
81
|
+
return GoogleCredential.fromStream(stream, httpTransport, jsonFactory)
|
82
|
+
.createScoped(Collections.singleton(StorageScopes.DEVSTORAGE_READ_WRITE));
|
83
|
+
}
|
84
|
+
|
71
85
|
/**
|
72
86
|
* @see http://developers.guge.io/accounts/docs/OAuth2ServiceAccount#creatinganaccount
|
73
87
|
* @see https://developers.google.com/accounts/docs/OAuth2
|
@@ -3,6 +3,7 @@ package org.embulk.output;
|
|
3
3
|
import com.google.api.client.http.InputStreamContent;
|
4
4
|
import com.google.api.services.storage.Storage;
|
5
5
|
import com.google.api.services.storage.model.StorageObject;
|
6
|
+
import com.google.common.base.Function;
|
6
7
|
import com.google.common.base.Optional;
|
7
8
|
import com.google.common.base.Throwables;
|
8
9
|
import org.embulk.config.TaskReport;
|
@@ -17,6 +18,7 @@ import org.embulk.spi.Buffer;
|
|
17
18
|
import org.embulk.spi.Exec;
|
18
19
|
import org.embulk.spi.FileOutputPlugin;
|
19
20
|
import org.embulk.spi.TransactionalFileOutput;
|
21
|
+
import org.embulk.spi.unit.LocalFile;
|
20
22
|
import org.slf4j.Logger;
|
21
23
|
|
22
24
|
import java.io.IOException;
|
@@ -60,10 +62,20 @@ public class GcsOutputPlugin implements FileOutputPlugin {
|
|
60
62
|
@ConfigDefault("null")
|
61
63
|
Optional<String> getServiceAccountEmail();
|
62
64
|
|
65
|
+
// kept for backward compatibility
|
63
66
|
@Config("p12_keyfile_path")
|
64
67
|
@ConfigDefault("null")
|
65
68
|
Optional<String> getP12KeyfilePath();
|
66
69
|
|
70
|
+
@Config("p12_keyfile")
|
71
|
+
@ConfigDefault("null")
|
72
|
+
Optional<LocalFile> getP12Keyfile();
|
73
|
+
void setP12Keyfile(Optional<LocalFile> p12Keyfile);
|
74
|
+
|
75
|
+
@Config("json_keyfile")
|
76
|
+
@ConfigDefault("null")
|
77
|
+
Optional<LocalFile> getJsonKeyfile();
|
78
|
+
|
67
79
|
@Config("application_name")
|
68
80
|
@ConfigDefault("\"embulk-output-gcs\"")
|
69
81
|
String getApplicationName();
|
@@ -74,6 +86,28 @@ public class GcsOutputPlugin implements FileOutputPlugin {
|
|
74
86
|
int taskCount,
|
75
87
|
FileOutputPlugin.Control control) {
|
76
88
|
PluginTask task = config.loadConfig(PluginTask.class);
|
89
|
+
|
90
|
+
if (task.getP12KeyfilePath().isPresent()) {
|
91
|
+
if (task.getP12Keyfile().isPresent()) {
|
92
|
+
throw new ConfigException("Setting both p12_keyfile_path and p12_keyfile is invalid");
|
93
|
+
}
|
94
|
+
try {
|
95
|
+
task.setP12Keyfile(Optional.of(LocalFile.of(task.getP12KeyfilePath().get())));
|
96
|
+
} catch (IOException ex) {
|
97
|
+
throw Throwables.propagate(ex);
|
98
|
+
}
|
99
|
+
}
|
100
|
+
|
101
|
+
if (task.getAuthMethod().getString().equals("json_key")) {
|
102
|
+
if (!task.getJsonKeyfile().isPresent()) {
|
103
|
+
throw new ConfigException("If auth_method is json_key, you have to set json_keyfile");
|
104
|
+
}
|
105
|
+
} else if (task.getAuthMethod().getString().equals("private_key")) {
|
106
|
+
if (!task.getP12Keyfile().isPresent() || !task.getServiceAccountEmail().isPresent()) {
|
107
|
+
throw new ConfigException("If auth_method is private_key, you have to set both service_account_email and p12_keyfile");
|
108
|
+
}
|
109
|
+
}
|
110
|
+
|
77
111
|
return resume(task.dump(), taskCount, control);
|
78
112
|
}
|
79
113
|
|
@@ -99,18 +133,43 @@ public class GcsOutputPlugin implements FileOutputPlugin {
|
|
99
133
|
return new TransactionalGcsFileOutput(task, client, taskIndex);
|
100
134
|
}
|
101
135
|
|
136
|
+
private GcsAuthentication newGcsAuth(PluginTask task)
|
137
|
+
{
|
138
|
+
try {
|
139
|
+
return new GcsAuthentication(
|
140
|
+
task.getAuthMethod().getString(),
|
141
|
+
task.getServiceAccountEmail(),
|
142
|
+
task.getP12Keyfile().transform(localFileToPathString()),
|
143
|
+
task.getJsonKeyfile().transform(localFileToPathString()),
|
144
|
+
task.getApplicationName()
|
145
|
+
);
|
146
|
+
} catch (GeneralSecurityException | IOException ex) {
|
147
|
+
throw new ConfigException(ex);
|
148
|
+
}
|
149
|
+
}
|
150
|
+
|
102
151
|
private Storage createClient(final PluginTask task) {
|
103
152
|
Storage client = null;
|
104
153
|
try {
|
105
|
-
GcsAuthentication auth =
|
154
|
+
GcsAuthentication auth = newGcsAuth(task);
|
106
155
|
client = auth.getGcsClient(task.getBucket());
|
107
|
-
} catch (
|
156
|
+
} catch (ConfigException | IOException ex) {
|
108
157
|
throw new ConfigException(ex);
|
109
158
|
}
|
110
159
|
|
111
160
|
return client;
|
112
161
|
}
|
113
162
|
|
163
|
+
private Function<LocalFile, String> localFileToPathString() {
|
164
|
+
return new Function<LocalFile, String>()
|
165
|
+
{
|
166
|
+
public String apply(LocalFile file)
|
167
|
+
{
|
168
|
+
return file.getPath().toString();
|
169
|
+
}
|
170
|
+
};
|
171
|
+
}
|
172
|
+
|
114
173
|
static class TransactionalGcsFileOutput implements TransactionalFileOutput {
|
115
174
|
private final int taskIndex;
|
116
175
|
private final Storage client;
|
@@ -233,7 +292,8 @@ public class GcsOutputPlugin implements FileOutputPlugin {
|
|
233
292
|
public enum AuthMethod
|
234
293
|
{
|
235
294
|
private_key("private_key"),
|
236
|
-
compute_engine("compute_engine")
|
295
|
+
compute_engine("compute_engine"),
|
296
|
+
json_key("json_key");
|
237
297
|
|
238
298
|
private final String string;
|
239
299
|
|
@@ -0,0 +1,177 @@
|
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.EmbulkTestRuntime;
|
5
|
+
import com.google.api.services.storage.Storage;
|
6
|
+
import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
|
7
|
+
|
8
|
+
import java.io.IOException;
|
9
|
+
import java.io.FileNotFoundException;
|
10
|
+
import java.security.GeneralSecurityException;
|
11
|
+
import com.google.api.client.googleapis.json.GoogleJsonResponseException;
|
12
|
+
|
13
|
+
import java.lang.reflect.Field;
|
14
|
+
import org.junit.BeforeClass;
|
15
|
+
import org.junit.Rule;
|
16
|
+
import org.junit.Test;
|
17
|
+
import static org.junit.Assert.assertEquals;
|
18
|
+
import static org.junit.Assume.assumeNotNull;
|
19
|
+
|
20
|
+
public class TestGcsAuthentication
|
21
|
+
{
|
22
|
+
private static Optional<String> GCP_EMAIL;
|
23
|
+
private static Optional<String> GCP_P12_KEYFILE;
|
24
|
+
private static Optional<String> GCP_JSON_KEYFILE;
|
25
|
+
private static String GCP_BUCKET;
|
26
|
+
private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
|
27
|
+
|
28
|
+
/*
|
29
|
+
* This test case requires environment variables
|
30
|
+
* GCP_EMAIL
|
31
|
+
* GCP_P12_KEYFILE
|
32
|
+
* GCP_JSON_KEYFILE
|
33
|
+
* GCP_BUCKET
|
34
|
+
*/
|
35
|
+
@BeforeClass
|
36
|
+
public static void initializeConstant()
|
37
|
+
{
|
38
|
+
GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
|
39
|
+
GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
|
40
|
+
GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
|
41
|
+
GCP_BUCKET = System.getenv("GCP_BUCKET");
|
42
|
+
// skip test cases, if environment variables are not set.
|
43
|
+
assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
|
44
|
+
}
|
45
|
+
|
46
|
+
@Rule
|
47
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
48
|
+
|
49
|
+
@Test
|
50
|
+
public void testGetServiceAccountCredentialSuccess()
|
51
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
52
|
+
{
|
53
|
+
GcsAuthentication auth = new GcsAuthentication(
|
54
|
+
"private_key",
|
55
|
+
GCP_EMAIL,
|
56
|
+
GCP_P12_KEYFILE,
|
57
|
+
null,
|
58
|
+
GCP_APPLICATION_NAME
|
59
|
+
);
|
60
|
+
|
61
|
+
Field field = GcsAuthentication.class.getDeclaredField("credentials");
|
62
|
+
field.setAccessible(true);
|
63
|
+
|
64
|
+
assertEquals(GoogleCredential.class, field.get(auth).getClass());
|
65
|
+
}
|
66
|
+
|
67
|
+
@Test(expected = FileNotFoundException.class)
|
68
|
+
public void testGetServiceAccountCredentialThrowFileNotFoundException()
|
69
|
+
throws GeneralSecurityException, IOException
|
70
|
+
{
|
71
|
+
Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
|
72
|
+
GcsAuthentication auth = new GcsAuthentication(
|
73
|
+
"private_key",
|
74
|
+
GCP_EMAIL,
|
75
|
+
notFoundP12Keyfile,
|
76
|
+
null,
|
77
|
+
GCP_APPLICATION_NAME
|
78
|
+
);
|
79
|
+
}
|
80
|
+
|
81
|
+
@Test
|
82
|
+
public void testGetGcsClientUsingServiceAccountCredentialSuccess()
|
83
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
84
|
+
{
|
85
|
+
GcsAuthentication auth = new GcsAuthentication(
|
86
|
+
"private_key",
|
87
|
+
GCP_EMAIL,
|
88
|
+
GCP_P12_KEYFILE,
|
89
|
+
null,
|
90
|
+
GCP_APPLICATION_NAME
|
91
|
+
);
|
92
|
+
|
93
|
+
Storage client = auth.getGcsClient(GCP_BUCKET);
|
94
|
+
|
95
|
+
assertEquals(Storage.class, client.getClass());
|
96
|
+
}
|
97
|
+
|
98
|
+
@Test(expected = GoogleJsonResponseException.class)
|
99
|
+
public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
|
100
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
101
|
+
{
|
102
|
+
GcsAuthentication auth = new GcsAuthentication(
|
103
|
+
"private_key",
|
104
|
+
GCP_EMAIL,
|
105
|
+
GCP_P12_KEYFILE,
|
106
|
+
null,
|
107
|
+
GCP_APPLICATION_NAME
|
108
|
+
);
|
109
|
+
|
110
|
+
Storage client = auth.getGcsClient("non-exists-bucket");
|
111
|
+
|
112
|
+
assertEquals(Storage.class, client.getClass());
|
113
|
+
}
|
114
|
+
|
115
|
+
@Test
|
116
|
+
public void testGetServiceAccountCredentialFromJsonFileSuccess()
|
117
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
118
|
+
{
|
119
|
+
GcsAuthentication auth = new GcsAuthentication(
|
120
|
+
"json_key",
|
121
|
+
GCP_EMAIL,
|
122
|
+
null,
|
123
|
+
GCP_JSON_KEYFILE,
|
124
|
+
GCP_APPLICATION_NAME
|
125
|
+
);
|
126
|
+
Field field = GcsAuthentication.class.getDeclaredField("credentials");
|
127
|
+
field.setAccessible(true);
|
128
|
+
|
129
|
+
assertEquals(GoogleCredential.class, field.get(auth).getClass());
|
130
|
+
}
|
131
|
+
|
132
|
+
@Test(expected = FileNotFoundException.class)
|
133
|
+
public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
|
134
|
+
throws GeneralSecurityException, IOException
|
135
|
+
{
|
136
|
+
Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
|
137
|
+
GcsAuthentication auth = new GcsAuthentication(
|
138
|
+
"json_key",
|
139
|
+
GCP_EMAIL,
|
140
|
+
null,
|
141
|
+
notFoundJsonKeyfile,
|
142
|
+
GCP_APPLICATION_NAME
|
143
|
+
);
|
144
|
+
}
|
145
|
+
|
146
|
+
@Test
|
147
|
+
public void testGetServiceAccountCredentialFromJsonSuccess()
|
148
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
149
|
+
{
|
150
|
+
GcsAuthentication auth = new GcsAuthentication(
|
151
|
+
"json_key",
|
152
|
+
GCP_EMAIL,
|
153
|
+
null,
|
154
|
+
GCP_JSON_KEYFILE,
|
155
|
+
GCP_APPLICATION_NAME
|
156
|
+
);
|
157
|
+
|
158
|
+
Storage client = auth.getGcsClient(GCP_BUCKET);
|
159
|
+
|
160
|
+
assertEquals(Storage.class, client.getClass());
|
161
|
+
}
|
162
|
+
|
163
|
+
@Test(expected = GoogleJsonResponseException.class)
|
164
|
+
public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
|
165
|
+
throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
|
166
|
+
{
|
167
|
+
GcsAuthentication auth = new GcsAuthentication(
|
168
|
+
"json_key",
|
169
|
+
GCP_EMAIL,
|
170
|
+
null,
|
171
|
+
GCP_JSON_KEYFILE,
|
172
|
+
GCP_APPLICATION_NAME
|
173
|
+
);
|
174
|
+
|
175
|
+
Storage client = auth.getGcsClient("non-exists-bucket");
|
176
|
+
}
|
177
|
+
}
|
@@ -1,5 +1,426 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
|
+
import java.io.BufferedReader;
|
4
|
+
import java.io.ByteArrayOutputStream;
|
5
|
+
import java.io.FileInputStream;
|
6
|
+
import java.io.InputStream;
|
7
|
+
import java.io.InputStreamReader;
|
8
|
+
import java.util.Arrays;
|
9
|
+
import java.util.List;
|
10
|
+
|
11
|
+
import com.google.common.collect.ImmutableMap;
|
12
|
+
import com.google.common.base.Optional;
|
13
|
+
import com.google.common.collect.ImmutableList;
|
14
|
+
import com.google.common.collect.Lists;
|
15
|
+
import java.io.IOException;
|
16
|
+
import java.security.GeneralSecurityException;
|
17
|
+
|
18
|
+
import org.embulk.EmbulkTestRuntime;
|
19
|
+
import org.embulk.config.TaskReport;
|
20
|
+
import org.embulk.config.TaskSource;
|
21
|
+
import org.embulk.config.ConfigDiff;
|
22
|
+
import org.embulk.config.ConfigSource;
|
23
|
+
import org.embulk.config.ConfigException;
|
24
|
+
import org.embulk.spi.Buffer;
|
25
|
+
import org.embulk.spi.Exec;
|
26
|
+
import org.embulk.spi.FileOutputPlugin;
|
27
|
+
import org.embulk.spi.FileOutputRunner;
|
28
|
+
import org.embulk.spi.OutputPlugin;
|
29
|
+
import org.embulk.spi.TransactionalFileOutput;
|
30
|
+
import org.embulk.spi.Schema;
|
31
|
+
import org.embulk.output.GcsOutputPlugin.PluginTask;
|
32
|
+
import org.embulk.standards.CsvParserPlugin;
|
33
|
+
|
34
|
+
import org.junit.BeforeClass;
|
35
|
+
import org.junit.Before;
|
36
|
+
import org.junit.Rule;
|
37
|
+
import org.junit.Test;
|
38
|
+
import static org.junit.Assert.assertEquals;
|
39
|
+
import static org.junit.Assume.assumeNotNull;
|
40
|
+
import java.lang.reflect.Method;
|
41
|
+
import java.lang.reflect.InvocationTargetException;
|
42
|
+
|
43
|
+
import com.google.api.services.storage.Storage;
|
44
|
+
|
3
45
|
public class TestGcsOutputPlugin
|
4
46
|
{
|
47
|
+
private static Optional<String> GCP_EMAIL;
|
48
|
+
private static Optional<String> GCP_P12_KEYFILE;
|
49
|
+
private static Optional<String> GCP_JSON_KEYFILE;
|
50
|
+
private static String GCP_BUCKET;
|
51
|
+
private static String GCP_BUCKET_DIRECTORY;
|
52
|
+
private static String GCP_PATH_PREFIX;
|
53
|
+
private static String LOCAL_PATH_PREFIX;
|
54
|
+
private final String GCP_APPLICATION_NAME = "embulk-output-gcs";
|
55
|
+
private FileOutputRunner runner;
|
56
|
+
|
57
|
+
/*
|
58
|
+
* This test case requires environment variables
|
59
|
+
* GCP_EMAIL
|
60
|
+
* GCP_P12_KEYFILE
|
61
|
+
* GCP_JSON_KEYFILE
|
62
|
+
* GCP_BUCKET
|
63
|
+
*/
|
64
|
+
@BeforeClass
|
65
|
+
public static void initializeConstant()
|
66
|
+
{
|
67
|
+
GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
|
68
|
+
GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
|
69
|
+
GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
|
70
|
+
GCP_BUCKET = System.getenv("GCP_BUCKET");
|
71
|
+
// skip test cases, if environment variables are not set.
|
72
|
+
assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
|
73
|
+
|
74
|
+
GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
|
75
|
+
GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
|
76
|
+
LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
|
77
|
+
}
|
78
|
+
|
79
|
+
@Rule
|
80
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
81
|
+
private GcsOutputPlugin plugin;
|
82
|
+
|
83
|
+
@Before
|
84
|
+
public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
|
85
|
+
{
|
86
|
+
plugin = new GcsOutputPlugin();
|
87
|
+
runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
|
88
|
+
}
|
89
|
+
|
90
|
+
@Test
|
91
|
+
public void checkDefaultValues()
|
92
|
+
{
|
93
|
+
ConfigSource config = Exec.newConfigSource()
|
94
|
+
.set("in", inputConfig())
|
95
|
+
.set("parser", parserConfig(schemaConfig()))
|
96
|
+
.set("type", "gcs")
|
97
|
+
.set("bucket", GCP_BUCKET)
|
98
|
+
.set("path_prefix", "my-prefix")
|
99
|
+
.set("file_ext", ".csv")
|
100
|
+
.set("formatter", formatterConfig());
|
101
|
+
|
102
|
+
GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
|
103
|
+
assertEquals("private_key", task.getAuthMethod().toString());
|
104
|
+
}
|
105
|
+
|
106
|
+
// p12_keyfile is null when auth_method is private_key
|
107
|
+
@Test(expected = ConfigException.class)
|
108
|
+
public void checkDefaultValuesP12keyNull()
|
109
|
+
{
|
110
|
+
ConfigSource config = Exec.newConfigSource()
|
111
|
+
.set("in", inputConfig())
|
112
|
+
.set("parser", parserConfig(schemaConfig()))
|
113
|
+
.set("type", "gcs")
|
114
|
+
.set("bucket", GCP_BUCKET)
|
115
|
+
.set("path_prefix", "my-prefix")
|
116
|
+
.set("file_ext", ".csv")
|
117
|
+
.set("auth_method", "private_key")
|
118
|
+
.set("service_account_email", GCP_EMAIL)
|
119
|
+
.set("p12_keyfile", null)
|
120
|
+
.set("formatter", formatterConfig());
|
121
|
+
|
122
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
123
|
+
|
124
|
+
runner.transaction(config, schema, 0, new Control());
|
125
|
+
}
|
126
|
+
|
127
|
+
// both p12_keyfile and p12_keyfile_path set
|
128
|
+
@Test(expected = ConfigException.class)
|
129
|
+
public void checkDefaultValuesConflictSetting()
|
130
|
+
{
|
131
|
+
ConfigSource config = Exec.newConfigSource()
|
132
|
+
.set("in", inputConfig())
|
133
|
+
.set("parser", parserConfig(schemaConfig()))
|
134
|
+
.set("type", "gcs")
|
135
|
+
.set("bucket", GCP_BUCKET)
|
136
|
+
.set("path_prefix", "my-prefix")
|
137
|
+
.set("file_ext", ".csv")
|
138
|
+
.set("auth_method", "private_key")
|
139
|
+
.set("service_account_email", GCP_EMAIL)
|
140
|
+
.set("p12_keyfile", GCP_P12_KEYFILE)
|
141
|
+
.set("p12_keyfile_path", GCP_P12_KEYFILE)
|
142
|
+
.set("formatter", formatterConfig());
|
143
|
+
|
144
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
145
|
+
|
146
|
+
runner.transaction(config, schema, 0, new Control());
|
147
|
+
}
|
148
|
+
|
149
|
+
// invalid p12keyfile when auth_method is private_key
|
150
|
+
@Test(expected = ConfigException.class)
|
151
|
+
public void checkDefaultValuesInvalidPrivateKey()
|
152
|
+
{
|
153
|
+
ConfigSource config = Exec.newConfigSource()
|
154
|
+
.set("in", inputConfig())
|
155
|
+
.set("parser", parserConfig(schemaConfig()))
|
156
|
+
.set("type", "gcs")
|
157
|
+
.set("bucket", GCP_BUCKET)
|
158
|
+
.set("path_prefix", "my-prefix")
|
159
|
+
.set("file_ext", ".csv")
|
160
|
+
.set("auth_method", "private_key")
|
161
|
+
.set("service_account_email", GCP_EMAIL)
|
162
|
+
.set("p12_keyfile", "invalid-key.p12")
|
163
|
+
.set("formatter", formatterConfig());
|
164
|
+
|
165
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
166
|
+
|
167
|
+
runner.transaction(config, schema, 0, new Control());
|
168
|
+
}
|
169
|
+
|
170
|
+
// json_keyfile is null when auth_method is json_key
|
171
|
+
@Test(expected = ConfigException.class)
|
172
|
+
public void checkDefaultValuesJsonKeyfileNull()
|
173
|
+
{
|
174
|
+
ConfigSource config = Exec.newConfigSource()
|
175
|
+
.set("in", inputConfig())
|
176
|
+
.set("parser", parserConfig(schemaConfig()))
|
177
|
+
.set("type", "gcs")
|
178
|
+
.set("bucket", GCP_BUCKET)
|
179
|
+
.set("path_prefix", "my-prefix")
|
180
|
+
.set("file_ext", ".csv")
|
181
|
+
.set("auth_method", "json_key")
|
182
|
+
.set("service_account_email", GCP_EMAIL)
|
183
|
+
.set("json_keyfile", null)
|
184
|
+
.set("formatter", formatterConfig());
|
185
|
+
|
186
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
187
|
+
|
188
|
+
runner.transaction(config, schema, 0, new Control());
|
189
|
+
}
|
190
|
+
|
191
|
+
@Test
|
192
|
+
public void testGcsClientCreateSuccessfully()
|
193
|
+
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
194
|
+
IllegalAccessException, InvocationTargetException
|
195
|
+
{
|
196
|
+
ConfigSource configSource = config();
|
197
|
+
PluginTask task = configSource.loadConfig(PluginTask.class);
|
198
|
+
Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
199
|
+
runner.transaction(configSource, schema, 0, new Control());
|
200
|
+
|
201
|
+
Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
202
|
+
method.setAccessible(true);
|
203
|
+
method.invoke(plugin, task); // no errors happens
|
204
|
+
}
|
205
|
+
|
206
|
+
@Test(expected = ConfigException.class)
|
207
|
+
public void testGcsClientCreateThrowConfigException()
|
208
|
+
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
209
|
+
IllegalAccessException, InvocationTargetException
|
210
|
+
{
|
211
|
+
ConfigSource config = Exec.newConfigSource()
|
212
|
+
.set("in", inputConfig())
|
213
|
+
.set("parser", parserConfig(schemaConfig()))
|
214
|
+
.set("type", "gcs")
|
215
|
+
.set("bucket", "non-exists-bucket")
|
216
|
+
.set("path_prefix", "my-prefix")
|
217
|
+
.set("file_ext", ".csv")
|
218
|
+
.set("auth_method", "json_key")
|
219
|
+
.set("service_account_email", GCP_EMAIL)
|
220
|
+
.set("json_keyfile", GCP_JSON_KEYFILE)
|
221
|
+
.set("formatter", formatterConfig());
|
222
|
+
|
223
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
224
|
+
|
225
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
226
|
+
runner.transaction(config, schema, 0, new Control());
|
227
|
+
|
228
|
+
Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
229
|
+
method.setAccessible(true);
|
230
|
+
try {
|
231
|
+
method.invoke(plugin, task);
|
232
|
+
} catch (InvocationTargetException ex) {
|
233
|
+
throw (ConfigException) ex.getCause();
|
234
|
+
}
|
235
|
+
}
|
236
|
+
|
237
|
+
@Test
|
238
|
+
public void testResume()
|
239
|
+
{
|
240
|
+
PluginTask task = config().loadConfig(PluginTask.class);
|
241
|
+
plugin.resume(task.dump(), 0, new FileOutputPlugin.Control() // no errors happens
|
242
|
+
{
|
243
|
+
@Override
|
244
|
+
public List<TaskReport> run(TaskSource taskSource)
|
245
|
+
{
|
246
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
247
|
+
}
|
248
|
+
});
|
249
|
+
}
|
250
|
+
|
251
|
+
@Test
|
252
|
+
public void testCleanup()
|
253
|
+
{
|
254
|
+
PluginTask task = config().loadConfig(PluginTask.class);
|
255
|
+
plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
|
256
|
+
}
|
257
|
+
|
258
|
+
@Test
|
259
|
+
public void testGcsFileOutputByOpen() throws Exception
|
260
|
+
{
|
261
|
+
ConfigSource configSource = config();
|
262
|
+
PluginTask task = configSource.loadConfig(PluginTask.class);
|
263
|
+
Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
264
|
+
runner.transaction(configSource, schema, 0, new Control());
|
265
|
+
|
266
|
+
TransactionalFileOutput output = plugin.open(task.dump(), 0);
|
267
|
+
|
268
|
+
output.nextFile();
|
269
|
+
|
270
|
+
FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
|
271
|
+
byte[] bytes = convertInputStreamToByte(is);
|
272
|
+
Buffer buffer = Buffer.wrap(bytes);
|
273
|
+
output.add(buffer);
|
274
|
+
|
275
|
+
output.finish();
|
276
|
+
output.commit();
|
277
|
+
|
278
|
+
String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
|
279
|
+
assertRecords(remotePath);
|
280
|
+
}
|
281
|
+
|
282
|
+
public ConfigSource config()
|
283
|
+
{
|
284
|
+
return Exec.newConfigSource()
|
285
|
+
.set("in", inputConfig())
|
286
|
+
.set("parser", parserConfig(schemaConfig()))
|
287
|
+
.set("type", "gcs")
|
288
|
+
.set("bucket", GCP_BUCKET)
|
289
|
+
.set("path_prefix", GCP_PATH_PREFIX)
|
290
|
+
.set("last_path", "")
|
291
|
+
.set("file_ext", ".csv")
|
292
|
+
.set("auth_method", "private_key")
|
293
|
+
.set("service_account_email", GCP_EMAIL)
|
294
|
+
.set("p12_keyfile", GCP_P12_KEYFILE)
|
295
|
+
.set("json_keyfile", GCP_JSON_KEYFILE)
|
296
|
+
.set("application_name", GCP_APPLICATION_NAME)
|
297
|
+
.set("formatter", formatterConfig());
|
298
|
+
}
|
299
|
+
|
300
|
+
private class Control
|
301
|
+
implements OutputPlugin.Control
|
302
|
+
{
|
303
|
+
@Override
|
304
|
+
public List<TaskReport> run(TaskSource taskSource)
|
305
|
+
{
|
306
|
+
return Lists.newArrayList(Exec.newTaskReport());
|
307
|
+
}
|
308
|
+
}
|
309
|
+
|
310
|
+
private ImmutableMap<String, Object> inputConfig()
|
311
|
+
{
|
312
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
313
|
+
builder.put("type", "file");
|
314
|
+
builder.put("path_prefix", LOCAL_PATH_PREFIX);
|
315
|
+
builder.put("last_path", "");
|
316
|
+
return builder.build();
|
317
|
+
}
|
318
|
+
|
319
|
+
private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
|
320
|
+
{
|
321
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
322
|
+
builder.put("type", "csv");
|
323
|
+
builder.put("newline", "CRLF");
|
324
|
+
builder.put("delimiter", ",");
|
325
|
+
builder.put("quote", "\"");
|
326
|
+
builder.put("escape", "\"");
|
327
|
+
builder.put("trim_if_not_quoted", false);
|
328
|
+
builder.put("skip_header_lines", 1);
|
329
|
+
builder.put("allow_extra_columns", false);
|
330
|
+
builder.put("allow_optional_columns", false);
|
331
|
+
builder.put("columns", schemaConfig);
|
332
|
+
return builder.build();
|
333
|
+
}
|
334
|
+
|
335
|
+
private ImmutableList<Object> schemaConfig()
|
336
|
+
{
|
337
|
+
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
338
|
+
builder.add(ImmutableMap.of("name", "id", "type", "long"));
|
339
|
+
builder.add(ImmutableMap.of("name", "account", "type", "long"));
|
340
|
+
builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
|
341
|
+
builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
|
342
|
+
builder.add(ImmutableMap.of("name", "comment", "type", "string"));
|
343
|
+
return builder.build();
|
344
|
+
}
|
345
|
+
|
346
|
+
private ImmutableMap<String, Object> formatterConfig()
|
347
|
+
{
|
348
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
349
|
+
builder.put("type", "csv");
|
350
|
+
builder.put("header_line", "false");
|
351
|
+
builder.put("timezone", "Asia/Tokyo");
|
352
|
+
return builder.build();
|
353
|
+
}
|
354
|
+
|
355
|
+
private void assertRecords(String gcsPath) throws Exception
|
356
|
+
{
|
357
|
+
ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
|
358
|
+
assertEquals(5, records.size());
|
359
|
+
{
|
360
|
+
List<String> record = records.get(1);
|
361
|
+
assertEquals("1", record.get(0));
|
362
|
+
assertEquals("32864", record.get(1));
|
363
|
+
assertEquals("2015-01-27 19:23:49", record.get(2));
|
364
|
+
assertEquals("20150127", record.get(3));
|
365
|
+
assertEquals("embulk", record.get(4));
|
366
|
+
}
|
367
|
+
|
368
|
+
{
|
369
|
+
List<String> record = records.get(2);
|
370
|
+
assertEquals("2", record.get(0));
|
371
|
+
assertEquals("14824", record.get(1));
|
372
|
+
assertEquals("2015-01-27 19:01:23", record.get(2));
|
373
|
+
assertEquals("20150127", record.get(3));
|
374
|
+
assertEquals("embulk jruby", record.get(4));
|
375
|
+
}
|
376
|
+
}
|
377
|
+
|
378
|
+
private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
|
379
|
+
{
|
380
|
+
ConfigSource config = config();
|
381
|
+
|
382
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
383
|
+
|
384
|
+
Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
|
385
|
+
method.setAccessible(true);
|
386
|
+
Storage client = (Storage) method.invoke(plugin, task);
|
387
|
+
Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
|
388
|
+
|
389
|
+
ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
|
390
|
+
|
391
|
+
InputStream is = getObject.executeMediaAsInputStream();
|
392
|
+
BufferedReader reader = new BufferedReader(new InputStreamReader(is));
|
393
|
+
String line;
|
394
|
+
while ((line = reader.readLine()) != null) {
|
395
|
+
List<String> records = Arrays.asList(line.split(",", 0));
|
396
|
+
|
397
|
+
builder.add(records);
|
398
|
+
}
|
399
|
+
return builder.build();
|
400
|
+
}
|
401
|
+
|
402
|
+
private static String getDirectory(String dir)
|
403
|
+
{
|
404
|
+
if (dir != null && !dir.endsWith("/")) {
|
405
|
+
dir = dir + "/";
|
406
|
+
}
|
407
|
+
if (dir.startsWith("/")) {
|
408
|
+
dir = dir.replaceFirst("/", "");
|
409
|
+
}
|
410
|
+
return dir;
|
411
|
+
}
|
412
|
+
|
413
|
+
private byte[] convertInputStreamToByte(InputStream is) throws IOException
|
414
|
+
{
|
415
|
+
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
416
|
+
byte [] buffer = new byte[1024];
|
417
|
+
while(true) {
|
418
|
+
int len = is.read(buffer);
|
419
|
+
if(len < 0) {
|
420
|
+
break;
|
421
|
+
}
|
422
|
+
bo.write(buffer, 0, len);
|
423
|
+
}
|
424
|
+
return bo.toByteArray();
|
425
|
+
}
|
5
426
|
}
|
Binary file
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-gcs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kazuyuki Honda
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -61,10 +62,14 @@ files:
|
|
61
62
|
- lib/embulk/output/gcs.rb
|
62
63
|
- src/main/java/org/embulk/output/GcsAuthentication.java
|
63
64
|
- src/main/java/org/embulk/output/GcsOutputPlugin.java
|
65
|
+
- src/test/java/org/embulk/output/TestGcsAuthentication.java
|
64
66
|
- src/test/java/org/embulk/output/TestGcsOutputPlugin.java
|
67
|
+
- src/test/resources/keys.tar.enc
|
68
|
+
- src/test/resources/sample_01.csv
|
69
|
+
- src/test/resources/sample_02.csv
|
65
70
|
- classpath/commons-codec-1.3.jar
|
66
71
|
- classpath/commons-logging-1.1.1.jar
|
67
|
-
- classpath/embulk-output-gcs-0.
|
72
|
+
- classpath/embulk-output-gcs-0.3.0.jar
|
68
73
|
- classpath/google-api-client-1.19.1.jar
|
69
74
|
- classpath/google-api-services-storage-v1-rev28-1.19.1.jar
|
70
75
|
- classpath/google-http-client-1.19.0.jar
|
Binary file
|