embulk-output-gcs 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/build.gradle +23 -5
  3. data/config/checkstyle/checkstyle.xml +130 -0
  4. data/config/checkstyle/default.xml +110 -0
  5. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  6. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  7. data/src/main/java/org/embulk/output/GcsAuthentication.java +184 -96
  8. data/src/main/java/org/embulk/output/GcsOutputPlugin.java +408 -278
  9. data/src/test/java/org/embulk/output/TestGcsAuthentication.java +161 -163
  10. data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java +399 -399
  11. metadata +5 -3
data/src/test/java/org/embulk/output/TestGcsAuthentication.java CHANGED
@@ -1,177 +1,175 @@
1
1
  package org.embulk.output;
2
2
 
3
- import com.google.common.base.Optional;
4
- import org.embulk.EmbulkTestRuntime;
5
- import com.google.api.services.storage.Storage;
6
3
  import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
4
+ import com.google.api.services.storage.Storage;
5
+ import com.google.common.base.Optional;
7
6
 
8
- import java.io.IOException;
9
- import java.io.FileNotFoundException;
10
- import java.security.GeneralSecurityException;
11
- import com.google.api.client.googleapis.json.GoogleJsonResponseException;
7
+ import org.embulk.EmbulkTestRuntime;
12
8
 
13
- import java.lang.reflect.Field;
9
+ import org.embulk.config.ConfigException;
14
10
  import org.junit.BeforeClass;
15
11
  import org.junit.Rule;
16
12
  import org.junit.Test;
17
13
  import static org.junit.Assert.assertEquals;
18
14
  import static org.junit.Assume.assumeNotNull;
19
15
 
16
+ import java.io.FileNotFoundException;
17
+
18
+ import java.io.IOException;
19
+ import java.lang.reflect.Field;
20
+ import java.security.GeneralSecurityException;
21
+
20
22
  public class TestGcsAuthentication
21
23
  {
22
- private static Optional<String> GCP_EMAIL;
23
- private static Optional<String> GCP_P12_KEYFILE;
24
- private static Optional<String> GCP_JSON_KEYFILE;
25
- private static String GCP_BUCKET;
26
- private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
27
-
28
- /*
29
- * This test case requires environment variables
30
- * GCP_EMAIL
31
- * GCP_P12_KEYFILE
32
- * GCP_JSON_KEYFILE
33
- * GCP_BUCKET
34
- */
35
- @BeforeClass
36
- public static void initializeConstant()
37
- {
38
- GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
39
- GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
40
- GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
41
- GCP_BUCKET = System.getenv("GCP_BUCKET");
42
- // skip test cases, if environment variables are not set.
43
- assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
44
- }
45
-
46
- @Rule
47
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
48
-
49
- @Test
50
- public void testGetServiceAccountCredentialSuccess()
51
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
52
- {
53
- GcsAuthentication auth = new GcsAuthentication(
54
- "private_key",
55
- GCP_EMAIL,
56
- GCP_P12_KEYFILE,
57
- null,
58
- GCP_APPLICATION_NAME
59
- );
60
-
61
- Field field = GcsAuthentication.class.getDeclaredField("credentials");
62
- field.setAccessible(true);
63
-
64
- assertEquals(GoogleCredential.class, field.get(auth).getClass());
65
- }
66
-
67
- @Test(expected = FileNotFoundException.class)
68
- public void testGetServiceAccountCredentialThrowFileNotFoundException()
69
- throws GeneralSecurityException, IOException
70
- {
71
- Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
72
- GcsAuthentication auth = new GcsAuthentication(
73
- "private_key",
74
- GCP_EMAIL,
75
- notFoundP12Keyfile,
76
- null,
77
- GCP_APPLICATION_NAME
78
- );
79
- }
80
-
81
- @Test
82
- public void testGetGcsClientUsingServiceAccountCredentialSuccess()
83
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
84
- {
85
- GcsAuthentication auth = new GcsAuthentication(
86
- "private_key",
87
- GCP_EMAIL,
88
- GCP_P12_KEYFILE,
89
- null,
90
- GCP_APPLICATION_NAME
91
- );
92
-
93
- Storage client = auth.getGcsClient(GCP_BUCKET);
94
-
95
- assertEquals(Storage.class, client.getClass());
96
- }
97
-
98
- @Test(expected = GoogleJsonResponseException.class)
99
- public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
100
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
101
- {
102
- GcsAuthentication auth = new GcsAuthentication(
103
- "private_key",
104
- GCP_EMAIL,
105
- GCP_P12_KEYFILE,
106
- null,
107
- GCP_APPLICATION_NAME
108
- );
109
-
110
- Storage client = auth.getGcsClient("non-exists-bucket");
111
-
112
- assertEquals(Storage.class, client.getClass());
113
- }
114
-
115
- @Test
116
- public void testGetServiceAccountCredentialFromJsonFileSuccess()
117
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
118
- {
119
- GcsAuthentication auth = new GcsAuthentication(
120
- "json_key",
121
- GCP_EMAIL,
122
- null,
123
- GCP_JSON_KEYFILE,
124
- GCP_APPLICATION_NAME
125
- );
126
- Field field = GcsAuthentication.class.getDeclaredField("credentials");
127
- field.setAccessible(true);
128
-
129
- assertEquals(GoogleCredential.class, field.get(auth).getClass());
130
- }
131
-
132
- @Test(expected = FileNotFoundException.class)
133
- public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
134
- throws GeneralSecurityException, IOException
135
- {
136
- Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
137
- GcsAuthentication auth = new GcsAuthentication(
138
- "json_key",
139
- GCP_EMAIL,
140
- null,
141
- notFoundJsonKeyfile,
142
- GCP_APPLICATION_NAME
143
- );
144
- }
145
-
146
- @Test
147
- public void testGetServiceAccountCredentialFromJsonSuccess()
148
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
149
- {
150
- GcsAuthentication auth = new GcsAuthentication(
151
- "json_key",
152
- GCP_EMAIL,
153
- null,
154
- GCP_JSON_KEYFILE,
155
- GCP_APPLICATION_NAME
156
- );
157
-
158
- Storage client = auth.getGcsClient(GCP_BUCKET);
159
-
160
- assertEquals(Storage.class, client.getClass());
161
- }
162
-
163
- @Test(expected = GoogleJsonResponseException.class)
164
- public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
165
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
166
- {
167
- GcsAuthentication auth = new GcsAuthentication(
168
- "json_key",
169
- GCP_EMAIL,
170
- null,
171
- GCP_JSON_KEYFILE,
172
- GCP_APPLICATION_NAME
173
- );
174
-
175
- Storage client = auth.getGcsClient("non-exists-bucket");
176
- }
24
+ private static Optional<String> GCP_EMAIL;
25
+ private static Optional<String> GCP_P12_KEYFILE;
26
+ private static Optional<String> GCP_JSON_KEYFILE;
27
+ private static String GCP_BUCKET;
28
+ private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
29
+
30
+ /*
31
+ * This test case requires environment variables
32
+ * GCP_EMAIL
33
+ * GCP_P12_KEYFILE
34
+ * GCP_JSON_KEYFILE
35
+ * GCP_BUCKET
36
+ */
37
+ @BeforeClass
38
+ public static void initializeConstant()
39
+ {
40
+ GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
41
+ GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
42
+ GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
43
+ GCP_BUCKET = System.getenv("GCP_BUCKET");
44
+ // skip test cases, if environment variables are not set.
45
+ assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
46
+ }
47
+
48
+ @Rule
49
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
50
+
51
+ @Test
52
+ public void testGetServiceAccountCredentialSuccess()
53
+ throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
54
+ {
55
+ GcsAuthentication auth = new GcsAuthentication(
56
+ "private_key",
57
+ GCP_EMAIL,
58
+ GCP_P12_KEYFILE,
59
+ null,
60
+ GCP_APPLICATION_NAME
61
+ );
62
+
63
+ Field field = GcsAuthentication.class.getDeclaredField("credentials");
64
+ field.setAccessible(true);
65
+
66
+ assertEquals(GoogleCredential.class, field.get(auth).getClass());
67
+ }
68
+
69
+ @Test(expected = FileNotFoundException.class)
70
+ public void testGetServiceAccountCredentialThrowFileNotFoundException()
71
+ throws GeneralSecurityException, IOException
72
+ {
73
+ Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
74
+ GcsAuthentication auth = new GcsAuthentication(
75
+ "private_key",
76
+ GCP_EMAIL,
77
+ notFoundP12Keyfile,
78
+ null,
79
+ GCP_APPLICATION_NAME
80
+ );
81
+ }
82
+
83
+ @Test
84
+ public void testGetGcsClientUsingServiceAccountCredentialSuccess() throws Exception
85
+ {
86
+ GcsAuthentication auth = new GcsAuthentication(
87
+ "private_key",
88
+ GCP_EMAIL,
89
+ GCP_P12_KEYFILE,
90
+ null,
91
+ GCP_APPLICATION_NAME
92
+ );
93
+
94
+ Storage client = auth.getGcsClient(GCP_BUCKET, 3);
95
+
96
+ assertEquals(Storage.class, client.getClass());
97
+ }
98
+
99
+ @Test(expected = ConfigException.class)
100
+ public void testGetGcsClientUsingServiceAccountCredentialThrowConfigException() throws Exception
101
+ {
102
+ GcsAuthentication auth = new GcsAuthentication(
103
+ "private_key",
104
+ GCP_EMAIL,
105
+ GCP_P12_KEYFILE,
106
+ null,
107
+ GCP_APPLICATION_NAME
108
+ );
109
+
110
+ Storage client = auth.getGcsClient("non-exists-bucket", 3);
111
+
112
+ assertEquals(Storage.class, client.getClass());
113
+ }
114
+
115
+ @Test
116
+ public void testGetServiceAccountCredentialFromJsonFileSuccess()
117
+ throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
118
+ {
119
+ GcsAuthentication auth = new GcsAuthentication(
120
+ "json_key",
121
+ GCP_EMAIL,
122
+ null,
123
+ GCP_JSON_KEYFILE,
124
+ GCP_APPLICATION_NAME
125
+ );
126
+ Field field = GcsAuthentication.class.getDeclaredField("credentials");
127
+ field.setAccessible(true);
128
+
129
+ assertEquals(GoogleCredential.class, field.get(auth).getClass());
130
+ }
131
+
132
+ @Test(expected = FileNotFoundException.class)
133
+ public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
134
+ throws GeneralSecurityException, IOException
135
+ {
136
+ Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
137
+ GcsAuthentication auth = new GcsAuthentication(
138
+ "json_key",
139
+ GCP_EMAIL,
140
+ null,
141
+ notFoundJsonKeyfile,
142
+ GCP_APPLICATION_NAME
143
+ );
144
+ }
145
+
146
+ @Test
147
+ public void testGetServiceAccountCredentialFromJsonSuccess() throws Exception
148
+ {
149
+ GcsAuthentication auth = new GcsAuthentication(
150
+ "json_key",
151
+ GCP_EMAIL,
152
+ null,
153
+ GCP_JSON_KEYFILE,
154
+ GCP_APPLICATION_NAME
155
+ );
156
+
157
+ Storage client = auth.getGcsClient(GCP_BUCKET, 3);
158
+
159
+ assertEquals(Storage.class, client.getClass());
160
+ }
161
+
162
+ @Test(expected = ConfigException.class)
163
+ public void testGetServiceAccountCredentialFromJsonThrowConfigException() throws Exception
164
+ {
165
+ GcsAuthentication auth = new GcsAuthentication(
166
+ "json_key",
167
+ GCP_EMAIL,
168
+ null,
169
+ GCP_JSON_KEYFILE,
170
+ GCP_APPLICATION_NAME
171
+ );
172
+
173
+ Storage client = auth.getGcsClient("non-exists-bucket", 3);
174
+ }
177
175
  }
data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java CHANGED
@@ -1,426 +1,426 @@
1
1
  package org.embulk.output;
2
2
 
3
- import java.io.BufferedReader;
4
- import java.io.ByteArrayOutputStream;
5
- import java.io.FileInputStream;
6
- import java.io.InputStream;
7
- import java.io.InputStreamReader;
8
- import java.util.Arrays;
9
- import java.util.List;
10
-
11
- import com.google.common.collect.ImmutableMap;
3
+ import com.google.api.services.storage.Storage;
12
4
  import com.google.common.base.Optional;
13
5
  import com.google.common.collect.ImmutableList;
6
+ import com.google.common.collect.ImmutableMap;
14
7
  import com.google.common.collect.Lists;
15
- import java.io.IOException;
16
- import java.security.GeneralSecurityException;
17
8
 
18
9
  import org.embulk.EmbulkTestRuntime;
10
+ import org.embulk.config.ConfigException;
11
+ import org.embulk.config.ConfigSource;
19
12
  import org.embulk.config.TaskReport;
20
13
  import org.embulk.config.TaskSource;
21
- import org.embulk.config.ConfigDiff;
22
- import org.embulk.config.ConfigSource;
23
- import org.embulk.config.ConfigException;
14
+ import org.embulk.output.GcsOutputPlugin.PluginTask;
24
15
  import org.embulk.spi.Buffer;
25
16
  import org.embulk.spi.Exec;
26
17
  import org.embulk.spi.FileOutputPlugin;
27
18
  import org.embulk.spi.FileOutputRunner;
28
19
  import org.embulk.spi.OutputPlugin;
29
- import org.embulk.spi.TransactionalFileOutput;
30
20
  import org.embulk.spi.Schema;
31
- import org.embulk.output.GcsOutputPlugin.PluginTask;
21
+ import org.embulk.spi.TransactionalFileOutput;
32
22
  import org.embulk.standards.CsvParserPlugin;
33
23
 
34
- import org.junit.BeforeClass;
35
24
  import org.junit.Before;
25
+ import org.junit.BeforeClass;
36
26
  import org.junit.Rule;
37
27
  import org.junit.Test;
38
28
  import static org.junit.Assert.assertEquals;
39
29
  import static org.junit.Assume.assumeNotNull;
40
- import java.lang.reflect.Method;
41
- import java.lang.reflect.InvocationTargetException;
42
30
 
43
- import com.google.api.services.storage.Storage;
31
+ import java.io.BufferedReader;
32
+ import java.io.ByteArrayOutputStream;
33
+ import java.io.FileInputStream;
34
+ import java.io.IOException;
35
+ import java.io.InputStream;
36
+ import java.io.InputStreamReader;
37
+ import java.lang.reflect.InvocationTargetException;
38
+ import java.lang.reflect.Method;
39
+ import java.security.GeneralSecurityException;
40
+ import java.util.Arrays;
41
+ import java.util.List;
44
42
 
45
43
  public class TestGcsOutputPlugin
46
44
  {
47
- private static Optional<String> GCP_EMAIL;
48
- private static Optional<String> GCP_P12_KEYFILE;
49
- private static Optional<String> GCP_JSON_KEYFILE;
50
- private static String GCP_BUCKET;
51
- private static String GCP_BUCKET_DIRECTORY;
52
- private static String GCP_PATH_PREFIX;
53
- private static String LOCAL_PATH_PREFIX;
54
- private final String GCP_APPLICATION_NAME = "embulk-output-gcs";
55
- private FileOutputRunner runner;
56
-
57
- /*
58
- * This test case requires environment variables
59
- * GCP_EMAIL
60
- * GCP_P12_KEYFILE
61
- * GCP_JSON_KEYFILE
62
- * GCP_BUCKET
63
- */
64
- @BeforeClass
65
- public static void initializeConstant()
66
- {
67
- GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
68
- GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
69
- GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
70
- GCP_BUCKET = System.getenv("GCP_BUCKET");
71
- // skip test cases, if environment variables are not set.
72
- assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
73
-
74
- GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
75
- GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
76
- LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
77
- }
78
-
79
- @Rule
80
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
81
- private GcsOutputPlugin plugin;
82
-
83
- @Before
84
- public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
85
- {
86
- plugin = new GcsOutputPlugin();
87
- runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
88
- }
89
-
90
- @Test
91
- public void checkDefaultValues()
92
- {
93
- ConfigSource config = Exec.newConfigSource()
94
- .set("in", inputConfig())
95
- .set("parser", parserConfig(schemaConfig()))
96
- .set("type", "gcs")
97
- .set("bucket", GCP_BUCKET)
98
- .set("path_prefix", "my-prefix")
99
- .set("file_ext", ".csv")
100
- .set("formatter", formatterConfig());
101
-
102
- GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
103
- assertEquals("private_key", task.getAuthMethod().toString());
104
- }
105
-
106
- // p12_keyfile is null when auth_method is private_key
107
- @Test(expected = ConfigException.class)
108
- public void checkDefaultValuesP12keyNull()
109
- {
110
- ConfigSource config = Exec.newConfigSource()
111
- .set("in", inputConfig())
112
- .set("parser", parserConfig(schemaConfig()))
113
- .set("type", "gcs")
114
- .set("bucket", GCP_BUCKET)
115
- .set("path_prefix", "my-prefix")
116
- .set("file_ext", ".csv")
117
- .set("auth_method", "private_key")
118
- .set("service_account_email", GCP_EMAIL)
119
- .set("p12_keyfile", null)
120
- .set("formatter", formatterConfig());
121
-
122
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
123
-
124
- runner.transaction(config, schema, 0, new Control());
125
- }
126
-
127
- // both p12_keyfile and p12_keyfile_path set
128
- @Test(expected = ConfigException.class)
129
- public void checkDefaultValuesConflictSetting()
130
- {
131
- ConfigSource config = Exec.newConfigSource()
132
- .set("in", inputConfig())
133
- .set("parser", parserConfig(schemaConfig()))
134
- .set("type", "gcs")
135
- .set("bucket", GCP_BUCKET)
136
- .set("path_prefix", "my-prefix")
137
- .set("file_ext", ".csv")
138
- .set("auth_method", "private_key")
139
- .set("service_account_email", GCP_EMAIL)
140
- .set("p12_keyfile", GCP_P12_KEYFILE)
141
- .set("p12_keyfile_path", GCP_P12_KEYFILE)
142
- .set("formatter", formatterConfig());
143
-
144
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
145
-
146
- runner.transaction(config, schema, 0, new Control());
147
- }
148
-
149
- // invalid p12keyfile when auth_method is private_key
150
- @Test(expected = ConfigException.class)
151
- public void checkDefaultValuesInvalidPrivateKey()
152
- {
153
- ConfigSource config = Exec.newConfigSource()
154
- .set("in", inputConfig())
155
- .set("parser", parserConfig(schemaConfig()))
156
- .set("type", "gcs")
157
- .set("bucket", GCP_BUCKET)
158
- .set("path_prefix", "my-prefix")
159
- .set("file_ext", ".csv")
160
- .set("auth_method", "private_key")
161
- .set("service_account_email", GCP_EMAIL)
162
- .set("p12_keyfile", "invalid-key.p12")
163
- .set("formatter", formatterConfig());
164
-
165
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
166
-
167
- runner.transaction(config, schema, 0, new Control());
168
- }
169
-
170
- // json_keyfile is null when auth_method is json_key
171
- @Test(expected = ConfigException.class)
172
- public void checkDefaultValuesJsonKeyfileNull()
173
- {
174
- ConfigSource config = Exec.newConfigSource()
175
- .set("in", inputConfig())
176
- .set("parser", parserConfig(schemaConfig()))
177
- .set("type", "gcs")
178
- .set("bucket", GCP_BUCKET)
179
- .set("path_prefix", "my-prefix")
180
- .set("file_ext", ".csv")
181
- .set("auth_method", "json_key")
182
- .set("service_account_email", GCP_EMAIL)
183
- .set("json_keyfile", null)
184
- .set("formatter", formatterConfig());
185
-
186
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
187
-
188
- runner.transaction(config, schema, 0, new Control());
189
- }
190
-
191
- @Test
192
- public void testGcsClientCreateSuccessfully()
193
- throws GeneralSecurityException, IOException, NoSuchMethodException,
194
- IllegalAccessException, InvocationTargetException
195
- {
196
- ConfigSource configSource = config();
197
- PluginTask task = configSource.loadConfig(PluginTask.class);
198
- Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
199
- runner.transaction(configSource, schema, 0, new Control());
200
-
201
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
202
- method.setAccessible(true);
203
- method.invoke(plugin, task); // no errors happens
204
- }
205
-
206
- @Test(expected = ConfigException.class)
207
- public void testGcsClientCreateThrowConfigException()
208
- throws GeneralSecurityException, IOException, NoSuchMethodException,
209
- IllegalAccessException, InvocationTargetException
210
- {
211
- ConfigSource config = Exec.newConfigSource()
212
- .set("in", inputConfig())
213
- .set("parser", parserConfig(schemaConfig()))
214
- .set("type", "gcs")
215
- .set("bucket", "non-exists-bucket")
216
- .set("path_prefix", "my-prefix")
217
- .set("file_ext", ".csv")
218
- .set("auth_method", "json_key")
219
- .set("service_account_email", GCP_EMAIL)
220
- .set("json_keyfile", GCP_JSON_KEYFILE)
221
- .set("formatter", formatterConfig());
222
-
223
- PluginTask task = config.loadConfig(PluginTask.class);
224
-
225
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
226
- runner.transaction(config, schema, 0, new Control());
227
-
228
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
229
- method.setAccessible(true);
230
- try {
231
- method.invoke(plugin, task);
232
- } catch (InvocationTargetException ex) {
233
- throw (ConfigException) ex.getCause();
234
- }
235
- }
236
-
237
- @Test
238
- public void testResume()
239
- {
240
- PluginTask task = config().loadConfig(PluginTask.class);
241
- plugin.resume(task.dump(), 0, new FileOutputPlugin.Control() // no errors happens
242
- {
243
- @Override
244
- public List<TaskReport> run(TaskSource taskSource)
245
- {
246
- return Lists.newArrayList(Exec.newTaskReport());
247
- }
248
- });
249
- }
250
-
251
- @Test
252
- public void testCleanup()
253
- {
254
- PluginTask task = config().loadConfig(PluginTask.class);
255
- plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
256
- }
257
-
258
- @Test
259
- public void testGcsFileOutputByOpen() throws Exception
260
- {
261
- ConfigSource configSource = config();
262
- PluginTask task = configSource.loadConfig(PluginTask.class);
263
- Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
264
- runner.transaction(configSource, schema, 0, new Control());
265
-
266
- TransactionalFileOutput output = plugin.open(task.dump(), 0);
267
-
268
- output.nextFile();
269
-
270
- FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
271
- byte[] bytes = convertInputStreamToByte(is);
272
- Buffer buffer = Buffer.wrap(bytes);
273
- output.add(buffer);
274
-
275
- output.finish();
276
- output.commit();
277
-
278
- String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
279
- assertRecords(remotePath);
280
- }
281
-
282
- public ConfigSource config()
283
- {
284
- return Exec.newConfigSource()
285
- .set("in", inputConfig())
286
- .set("parser", parserConfig(schemaConfig()))
287
- .set("type", "gcs")
288
- .set("bucket", GCP_BUCKET)
289
- .set("path_prefix", GCP_PATH_PREFIX)
290
- .set("last_path", "")
291
- .set("file_ext", ".csv")
292
- .set("auth_method", "private_key")
293
- .set("service_account_email", GCP_EMAIL)
294
- .set("p12_keyfile", GCP_P12_KEYFILE)
295
- .set("json_keyfile", GCP_JSON_KEYFILE)
296
- .set("application_name", GCP_APPLICATION_NAME)
297
- .set("formatter", formatterConfig());
298
- }
299
-
300
- private class Control
301
- implements OutputPlugin.Control
302
- {
303
- @Override
304
- public List<TaskReport> run(TaskSource taskSource)
305
- {
306
- return Lists.newArrayList(Exec.newTaskReport());
307
- }
308
- }
309
-
310
- private ImmutableMap<String, Object> inputConfig()
311
- {
312
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
313
- builder.put("type", "file");
314
- builder.put("path_prefix", LOCAL_PATH_PREFIX);
315
- builder.put("last_path", "");
316
- return builder.build();
317
- }
318
-
319
- private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
320
- {
321
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
322
- builder.put("type", "csv");
323
- builder.put("newline", "CRLF");
324
- builder.put("delimiter", ",");
325
- builder.put("quote", "\"");
326
- builder.put("escape", "\"");
327
- builder.put("trim_if_not_quoted", false);
328
- builder.put("skip_header_lines", 1);
329
- builder.put("allow_extra_columns", false);
330
- builder.put("allow_optional_columns", false);
331
- builder.put("columns", schemaConfig);
332
- return builder.build();
333
- }
334
-
335
- private ImmutableList<Object> schemaConfig()
336
- {
337
- ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
338
- builder.add(ImmutableMap.of("name", "id", "type", "long"));
339
- builder.add(ImmutableMap.of("name", "account", "type", "long"));
340
- builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
341
- builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
342
- builder.add(ImmutableMap.of("name", "comment", "type", "string"));
343
- return builder.build();
344
- }
345
-
346
- private ImmutableMap<String, Object> formatterConfig()
347
- {
348
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
349
- builder.put("type", "csv");
350
- builder.put("header_line", "false");
351
- builder.put("timezone", "Asia/Tokyo");
352
- return builder.build();
353
- }
354
-
355
- private void assertRecords(String gcsPath) throws Exception
356
- {
357
- ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
358
- assertEquals(5, records.size());
359
- {
360
- List<String> record = records.get(1);
361
- assertEquals("1", record.get(0));
362
- assertEquals("32864", record.get(1));
363
- assertEquals("2015-01-27 19:23:49", record.get(2));
364
- assertEquals("20150127", record.get(3));
365
- assertEquals("embulk", record.get(4));
366
- }
367
-
368
- {
369
- List<String> record = records.get(2);
370
- assertEquals("2", record.get(0));
371
- assertEquals("14824", record.get(1));
372
- assertEquals("2015-01-27 19:01:23", record.get(2));
373
- assertEquals("20150127", record.get(3));
374
- assertEquals("embulk jruby", record.get(4));
375
- }
376
- }
377
-
378
- private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
379
- {
380
- ConfigSource config = config();
381
-
382
- PluginTask task = config.loadConfig(PluginTask.class);
383
-
384
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
385
- method.setAccessible(true);
386
- Storage client = (Storage) method.invoke(plugin, task);
387
- Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
388
-
389
- ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
390
-
391
- InputStream is = getObject.executeMediaAsInputStream();
392
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
393
- String line;
394
- while ((line = reader.readLine()) != null) {
395
- List<String> records = Arrays.asList(line.split(",", 0));
396
-
397
- builder.add(records);
398
- }
399
- return builder.build();
400
- }
401
-
402
- private static String getDirectory(String dir)
403
- {
404
- if (dir != null && !dir.endsWith("/")) {
405
- dir = dir + "/";
406
- }
407
- if (dir.startsWith("/")) {
408
- dir = dir.replaceFirst("/", "");
409
- }
410
- return dir;
411
- }
412
-
413
- private byte[] convertInputStreamToByte(InputStream is) throws IOException
414
- {
415
- ByteArrayOutputStream bo = new ByteArrayOutputStream();
416
- byte [] buffer = new byte[1024];
417
- while(true) {
418
- int len = is.read(buffer);
419
- if(len < 0) {
420
- break;
421
- }
422
- bo.write(buffer, 0, len);
423
- }
424
- return bo.toByteArray();
425
- }
45
+ private static Optional<String> GCP_EMAIL;
46
+ private static Optional<String> GCP_P12_KEYFILE;
47
+ private static Optional<String> GCP_JSON_KEYFILE;
48
+ private static String GCP_BUCKET;
49
+ private static String GCP_BUCKET_DIRECTORY;
50
+ private static String GCP_PATH_PREFIX;
51
+ private static String LOCAL_PATH_PREFIX;
52
+ private static String GCP_APPLICATION_NAME;
53
+ private FileOutputRunner runner;
54
+
55
+ /*
56
+ * This test case requires environment variables
57
+ * GCP_EMAIL
58
+ * GCP_P12_KEYFILE
59
+ * GCP_JSON_KEYFILE
60
+ * GCP_BUCKET
61
+ */
62
+ @BeforeClass
63
+ public static void initializeConstant()
64
+ {
65
+ GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
66
+ GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
67
+ GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
68
+ GCP_BUCKET = System.getenv("GCP_BUCKET");
69
+ // skip test cases, if environment variables are not set.
70
+ assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
71
+
72
+ GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
73
+ GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
74
+ LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
75
+ GCP_APPLICATION_NAME = "embulk-output-gcs";
76
+ }
77
+
78
+ @Rule
79
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
80
+ private GcsOutputPlugin plugin;
81
+
82
+ @Before
83
+ public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
84
+ {
85
+ plugin = new GcsOutputPlugin();
86
+ runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
87
+ }
88
+
89
+ @Test
90
+ public void checkDefaultValues()
91
+ {
92
+ ConfigSource config = Exec.newConfigSource()
93
+ .set("in", inputConfig())
94
+ .set("parser", parserConfig(schemaConfig()))
95
+ .set("type", "gcs")
96
+ .set("bucket", GCP_BUCKET)
97
+ .set("path_prefix", "my-prefix")
98
+ .set("file_ext", ".csv")
99
+ .set("formatter", formatterConfig());
100
+
101
+ GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
102
+ assertEquals("private_key", task.getAuthMethod().toString());
103
+ }
104
+
105
+ // p12_keyfile is null when auth_method is private_key
106
+ @Test(expected = ConfigException.class)
107
+ public void checkDefaultValuesP12keyNull()
108
+ {
109
+ ConfigSource config = Exec.newConfigSource()
110
+ .set("in", inputConfig())
111
+ .set("parser", parserConfig(schemaConfig()))
112
+ .set("type", "gcs")
113
+ .set("bucket", GCP_BUCKET)
114
+ .set("path_prefix", "my-prefix")
115
+ .set("file_ext", ".csv")
116
+ .set("auth_method", "private_key")
117
+ .set("service_account_email", GCP_EMAIL)
118
+ .set("p12_keyfile", null)
119
+ .set("formatter", formatterConfig());
120
+
121
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
122
+
123
+ runner.transaction(config, schema, 0, new Control());
124
+ }
125
+
126
+ // both p12_keyfile and p12_keyfile_path set
127
+ @Test(expected = ConfigException.class)
128
+ public void checkDefaultValuesConflictSetting()
129
+ {
130
+ ConfigSource config = Exec.newConfigSource()
131
+ .set("in", inputConfig())
132
+ .set("parser", parserConfig(schemaConfig()))
133
+ .set("type", "gcs")
134
+ .set("bucket", GCP_BUCKET)
135
+ .set("path_prefix", "my-prefix")
136
+ .set("file_ext", ".csv")
137
+ .set("auth_method", "private_key")
138
+ .set("service_account_email", GCP_EMAIL)
139
+ .set("p12_keyfile", GCP_P12_KEYFILE)
140
+ .set("p12_keyfile_path", GCP_P12_KEYFILE)
141
+ .set("formatter", formatterConfig());
142
+
143
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
144
+
145
+ runner.transaction(config, schema, 0, new Control());
146
+ }
147
+
148
+ // invalid p12keyfile when auth_method is private_key
149
+ @Test(expected = ConfigException.class)
150
+ public void checkDefaultValuesInvalidPrivateKey()
151
+ {
152
+ ConfigSource config = Exec.newConfigSource()
153
+ .set("in", inputConfig())
154
+ .set("parser", parserConfig(schemaConfig()))
155
+ .set("type", "gcs")
156
+ .set("bucket", GCP_BUCKET)
157
+ .set("path_prefix", "my-prefix")
158
+ .set("file_ext", ".csv")
159
+ .set("auth_method", "private_key")
160
+ .set("service_account_email", GCP_EMAIL)
161
+ .set("p12_keyfile", "invalid-key.p12")
162
+ .set("formatter", formatterConfig());
163
+
164
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
165
+
166
+ runner.transaction(config, schema, 0, new Control());
167
+ }
168
+
169
+ // json_keyfile is null when auth_method is json_key
170
+ @Test(expected = ConfigException.class)
171
+ public void checkDefaultValuesJsonKeyfileNull()
172
+ {
173
+ ConfigSource config = Exec.newConfigSource()
174
+ .set("in", inputConfig())
175
+ .set("parser", parserConfig(schemaConfig()))
176
+ .set("type", "gcs")
177
+ .set("bucket", GCP_BUCKET)
178
+ .set("path_prefix", "my-prefix")
179
+ .set("file_ext", ".csv")
180
+ .set("auth_method", "json_key")
181
+ .set("service_account_email", GCP_EMAIL)
182
+ .set("json_keyfile", null)
183
+ .set("formatter", formatterConfig());
184
+
185
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
186
+
187
+ runner.transaction(config, schema, 0, new Control());
188
+ }
189
+
190
+ @Test
191
+ public void testGcsClientCreateSuccessfully()
192
+ throws GeneralSecurityException, IOException, NoSuchMethodException,
193
+ IllegalAccessException, InvocationTargetException
194
+ {
195
+ ConfigSource configSource = config();
196
+ PluginTask task = configSource.loadConfig(PluginTask.class);
197
+ Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
198
+ runner.transaction(configSource, schema, 0, new Control());
199
+
200
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
201
+ method.setAccessible(true);
202
+ method.invoke(plugin, task); // no errors happens
203
+ }
204
+
205
+ @Test(expected = ConfigException.class)
206
+ public void testGcsClientCreateThrowConfigException()
207
+ throws GeneralSecurityException, IOException, NoSuchMethodException,
208
+ IllegalAccessException, InvocationTargetException
209
+ {
210
+ ConfigSource config = Exec.newConfigSource()
211
+ .set("in", inputConfig())
212
+ .set("parser", parserConfig(schemaConfig()))
213
+ .set("type", "gcs")
214
+ .set("bucket", "non-exists-bucket")
215
+ .set("path_prefix", "my-prefix")
216
+ .set("file_ext", ".csv")
217
+ .set("auth_method", "json_key")
218
+ .set("service_account_email", GCP_EMAIL)
219
+ .set("json_keyfile", GCP_JSON_KEYFILE)
220
+ .set("formatter", formatterConfig());
221
+
222
+ PluginTask task = config.loadConfig(PluginTask.class);
223
+
224
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
225
+ runner.transaction(config, schema, 0, new Control());
226
+
227
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
228
+ method.setAccessible(true);
229
+ try {
230
+ method.invoke(plugin, task);
231
+ }
232
+ catch (InvocationTargetException ex) {
233
+ throw (ConfigException) ex.getCause();
234
+ }
235
+ }
236
+
237
+ @Test
238
+ public void testResume()
239
+ {
240
+ PluginTask task = config().loadConfig(PluginTask.class);
241
+ plugin.resume(task.dump(), 0, new FileOutputPlugin.Control() // no errors happens
242
+ {
243
+ @Override
244
+ public List<TaskReport> run(TaskSource taskSource)
245
+ {
246
+ return Lists.newArrayList(Exec.newTaskReport());
247
+ }
248
+ });
249
+ }
250
+
251
+ @Test
252
+ public void testCleanup()
253
+ {
254
+ PluginTask task = config().loadConfig(PluginTask.class);
255
+ plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
256
+ }
257
+
258
+ @Test
259
+ public void testGcsFileOutputByOpen() throws Exception
260
+ {
261
+ ConfigSource configSource = config();
262
+ PluginTask task = configSource.loadConfig(PluginTask.class);
263
+ Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
264
+ runner.transaction(configSource, schema, 0, new Control());
265
+
266
+ TransactionalFileOutput output = plugin.open(task.dump(), 0);
267
+
268
+ output.nextFile();
269
+
270
+ FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
271
+ byte[] bytes = convertInputStreamToByte(is);
272
+ Buffer buffer = Buffer.wrap(bytes);
273
+ output.add(buffer);
274
+
275
+ output.finish();
276
+ output.commit();
277
+
278
+ String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
279
+ assertRecords(remotePath);
280
+ }
281
+
282
+ public ConfigSource config()
283
+ {
284
+ return Exec.newConfigSource()
285
+ .set("in", inputConfig())
286
+ .set("parser", parserConfig(schemaConfig()))
287
+ .set("type", "gcs")
288
+ .set("bucket", GCP_BUCKET)
289
+ .set("path_prefix", GCP_PATH_PREFIX)
290
+ .set("last_path", "")
291
+ .set("file_ext", ".csv")
292
+ .set("auth_method", "private_key")
293
+ .set("service_account_email", GCP_EMAIL)
294
+ .set("p12_keyfile", GCP_P12_KEYFILE)
295
+ .set("json_keyfile", GCP_JSON_KEYFILE)
296
+ .set("application_name", GCP_APPLICATION_NAME)
297
+ .set("formatter", formatterConfig());
298
+ }
299
+
300
+ private class Control
301
+ implements OutputPlugin.Control
302
+ {
303
+ @Override
304
+ public List<TaskReport> run(TaskSource taskSource)
305
+ {
306
+ return Lists.newArrayList(Exec.newTaskReport());
307
+ }
308
+ }
309
+
310
+ private ImmutableMap<String, Object> inputConfig()
311
+ {
312
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
313
+ builder.put("type", "file");
314
+ builder.put("path_prefix", LOCAL_PATH_PREFIX);
315
+ builder.put("last_path", "");
316
+ return builder.build();
317
+ }
318
+
319
+ private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
320
+ {
321
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
322
+ builder.put("type", "csv");
323
+ builder.put("newline", "CRLF");
324
+ builder.put("delimiter", ",");
325
+ builder.put("quote", "\"");
326
+ builder.put("escape", "\"");
327
+ builder.put("trim_if_not_quoted", false);
328
+ builder.put("skip_header_lines", 1);
329
+ builder.put("allow_extra_columns", false);
330
+ builder.put("allow_optional_columns", false);
331
+ builder.put("columns", schemaConfig);
332
+ return builder.build();
333
+ }
334
+
335
+ private ImmutableList<Object> schemaConfig()
336
+ {
337
+ ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
338
+ builder.add(ImmutableMap.of("name", "id", "type", "long"));
339
+ builder.add(ImmutableMap.of("name", "account", "type", "long"));
340
+ builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
341
+ builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
342
+ builder.add(ImmutableMap.of("name", "comment", "type", "string"));
343
+ return builder.build();
344
+ }
345
+
346
+ private ImmutableMap<String, Object> formatterConfig()
347
+ {
348
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
349
+ builder.put("type", "csv");
350
+ builder.put("header_line", "false");
351
+ builder.put("timezone", "Asia/Tokyo");
352
+ return builder.build();
353
+ }
354
+
355
+ private void assertRecords(String gcsPath) throws Exception
356
+ {
357
+ ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
358
+ assertEquals(5, records.size());
359
+ {
360
+ List<String> record = records.get(1);
361
+ assertEquals("1", record.get(0));
362
+ assertEquals("32864", record.get(1));
363
+ assertEquals("2015-01-27 19:23:49", record.get(2));
364
+ assertEquals("20150127", record.get(3));
365
+ assertEquals("embulk", record.get(4));
366
+ }
367
+
368
+ {
369
+ List<String> record = records.get(2);
370
+ assertEquals("2", record.get(0));
371
+ assertEquals("14824", record.get(1));
372
+ assertEquals("2015-01-27 19:01:23", record.get(2));
373
+ assertEquals("20150127", record.get(3));
374
+ assertEquals("embulk jruby", record.get(4));
375
+ }
376
+ }
377
+
378
+ private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
379
+ {
380
+ ConfigSource config = config();
381
+
382
+ PluginTask task = config.loadConfig(PluginTask.class);
383
+
384
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
385
+ method.setAccessible(true);
386
+ Storage client = (Storage) method.invoke(plugin, task);
387
+ Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
388
+
389
+ ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
390
+
391
+ InputStream is = getObject.executeMediaAsInputStream();
392
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is));
393
+ String line;
394
+ while ((line = reader.readLine()) != null) {
395
+ List<String> records = Arrays.asList(line.split(",", 0));
396
+
397
+ builder.add(records);
398
+ }
399
+ return builder.build();
400
+ }
401
+
402
+ private static String getDirectory(String dir)
403
+ {
404
+ if (dir != null && !dir.endsWith("/")) {
405
+ dir = dir + "/";
406
+ }
407
+ if (dir.startsWith("/")) {
408
+ dir = dir.replaceFirst("/", "");
409
+ }
410
+ return dir;
411
+ }
412
+
413
+ private byte[] convertInputStreamToByte(InputStream is) throws IOException
414
+ {
415
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
416
+ byte [] buffer = new byte[1024];
417
+ while (true) {
418
+ int len = is.read(buffer);
419
+ if (len < 0) {
420
+ break;
421
+ }
422
+ bo.write(buffer, 0, len);
423
+ }
424
+ return bo.toByteArray();
425
+ }
426
426
  }