embulk-output-gcs 0.3.0 → 0.4.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/build.gradle +23 -5
  3. data/config/checkstyle/checkstyle.xml +130 -0
  4. data/config/checkstyle/default.xml +110 -0
  5. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  6. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  7. data/src/main/java/org/embulk/output/GcsAuthentication.java +184 -96
  8. data/src/main/java/org/embulk/output/GcsOutputPlugin.java +408 -278
  9. data/src/test/java/org/embulk/output/TestGcsAuthentication.java +161 -163
  10. data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java +399 -399
  11. metadata +5 -3
data/src/test/java/org/embulk/output/TestGcsAuthentication.java CHANGED
@@ -1,177 +1,175 @@
1
1
  package org.embulk.output;
2
2
 
3
- import com.google.common.base.Optional;
4
- import org.embulk.EmbulkTestRuntime;
5
- import com.google.api.services.storage.Storage;
6
3
  import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
4
+ import com.google.api.services.storage.Storage;
5
+ import com.google.common.base.Optional;
7
6
 
8
- import java.io.IOException;
9
- import java.io.FileNotFoundException;
10
- import java.security.GeneralSecurityException;
11
- import com.google.api.client.googleapis.json.GoogleJsonResponseException;
7
+ import org.embulk.EmbulkTestRuntime;
12
8
 
13
- import java.lang.reflect.Field;
9
+ import org.embulk.config.ConfigException;
14
10
  import org.junit.BeforeClass;
15
11
  import org.junit.Rule;
16
12
  import org.junit.Test;
17
13
  import static org.junit.Assert.assertEquals;
18
14
  import static org.junit.Assume.assumeNotNull;
19
15
 
16
+ import java.io.FileNotFoundException;
17
+
18
+ import java.io.IOException;
19
+ import java.lang.reflect.Field;
20
+ import java.security.GeneralSecurityException;
21
+
20
22
  public class TestGcsAuthentication
21
23
  {
22
- private static Optional<String> GCP_EMAIL;
23
- private static Optional<String> GCP_P12_KEYFILE;
24
- private static Optional<String> GCP_JSON_KEYFILE;
25
- private static String GCP_BUCKET;
26
- private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
27
-
28
- /*
29
- * This test case requires environment variables
30
- * GCP_EMAIL
31
- * GCP_P12_KEYFILE
32
- * GCP_JSON_KEYFILE
33
- * GCP_BUCKET
34
- */
35
- @BeforeClass
36
- public static void initializeConstant()
37
- {
38
- GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
39
- GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
40
- GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
41
- GCP_BUCKET = System.getenv("GCP_BUCKET");
42
- // skip test cases, if environment variables are not set.
43
- assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
44
- }
45
-
46
- @Rule
47
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
48
-
49
- @Test
50
- public void testGetServiceAccountCredentialSuccess()
51
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
52
- {
53
- GcsAuthentication auth = new GcsAuthentication(
54
- "private_key",
55
- GCP_EMAIL,
56
- GCP_P12_KEYFILE,
57
- null,
58
- GCP_APPLICATION_NAME
59
- );
60
-
61
- Field field = GcsAuthentication.class.getDeclaredField("credentials");
62
- field.setAccessible(true);
63
-
64
- assertEquals(GoogleCredential.class, field.get(auth).getClass());
65
- }
66
-
67
- @Test(expected = FileNotFoundException.class)
68
- public void testGetServiceAccountCredentialThrowFileNotFoundException()
69
- throws GeneralSecurityException, IOException
70
- {
71
- Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
72
- GcsAuthentication auth = new GcsAuthentication(
73
- "private_key",
74
- GCP_EMAIL,
75
- notFoundP12Keyfile,
76
- null,
77
- GCP_APPLICATION_NAME
78
- );
79
- }
80
-
81
- @Test
82
- public void testGetGcsClientUsingServiceAccountCredentialSuccess()
83
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
84
- {
85
- GcsAuthentication auth = new GcsAuthentication(
86
- "private_key",
87
- GCP_EMAIL,
88
- GCP_P12_KEYFILE,
89
- null,
90
- GCP_APPLICATION_NAME
91
- );
92
-
93
- Storage client = auth.getGcsClient(GCP_BUCKET);
94
-
95
- assertEquals(Storage.class, client.getClass());
96
- }
97
-
98
- @Test(expected = GoogleJsonResponseException.class)
99
- public void testGetGcsClientUsingServiceAccountCredentialThrowJsonResponseException()
100
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
101
- {
102
- GcsAuthentication auth = new GcsAuthentication(
103
- "private_key",
104
- GCP_EMAIL,
105
- GCP_P12_KEYFILE,
106
- null,
107
- GCP_APPLICATION_NAME
108
- );
109
-
110
- Storage client = auth.getGcsClient("non-exists-bucket");
111
-
112
- assertEquals(Storage.class, client.getClass());
113
- }
114
-
115
- @Test
116
- public void testGetServiceAccountCredentialFromJsonFileSuccess()
117
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
118
- {
119
- GcsAuthentication auth = new GcsAuthentication(
120
- "json_key",
121
- GCP_EMAIL,
122
- null,
123
- GCP_JSON_KEYFILE,
124
- GCP_APPLICATION_NAME
125
- );
126
- Field field = GcsAuthentication.class.getDeclaredField("credentials");
127
- field.setAccessible(true);
128
-
129
- assertEquals(GoogleCredential.class, field.get(auth).getClass());
130
- }
131
-
132
- @Test(expected = FileNotFoundException.class)
133
- public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
134
- throws GeneralSecurityException, IOException
135
- {
136
- Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
137
- GcsAuthentication auth = new GcsAuthentication(
138
- "json_key",
139
- GCP_EMAIL,
140
- null,
141
- notFoundJsonKeyfile,
142
- GCP_APPLICATION_NAME
143
- );
144
- }
145
-
146
- @Test
147
- public void testGetServiceAccountCredentialFromJsonSuccess()
148
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
149
- {
150
- GcsAuthentication auth = new GcsAuthentication(
151
- "json_key",
152
- GCP_EMAIL,
153
- null,
154
- GCP_JSON_KEYFILE,
155
- GCP_APPLICATION_NAME
156
- );
157
-
158
- Storage client = auth.getGcsClient(GCP_BUCKET);
159
-
160
- assertEquals(Storage.class, client.getClass());
161
- }
162
-
163
- @Test(expected = GoogleJsonResponseException.class)
164
- public void testGetServiceAccountCredentialFromJsonThrowGoogleJsonResponseException()
165
- throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
166
- {
167
- GcsAuthentication auth = new GcsAuthentication(
168
- "json_key",
169
- GCP_EMAIL,
170
- null,
171
- GCP_JSON_KEYFILE,
172
- GCP_APPLICATION_NAME
173
- );
174
-
175
- Storage client = auth.getGcsClient("non-exists-bucket");
176
- }
24
+ private static Optional<String> GCP_EMAIL;
25
+ private static Optional<String> GCP_P12_KEYFILE;
26
+ private static Optional<String> GCP_JSON_KEYFILE;
27
+ private static String GCP_BUCKET;
28
+ private static final String GCP_APPLICATION_NAME = "embulk-output-gcs";
29
+
30
+ /*
31
+ * This test case requires environment variables
32
+ * GCP_EMAIL
33
+ * GCP_P12_KEYFILE
34
+ * GCP_JSON_KEYFILE
35
+ * GCP_BUCKET
36
+ */
37
+ @BeforeClass
38
+ public static void initializeConstant()
39
+ {
40
+ GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
41
+ GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
42
+ GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
43
+ GCP_BUCKET = System.getenv("GCP_BUCKET");
44
+ // skip test cases, if environment variables are not set.
45
+ assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
46
+ }
47
+
48
+ @Rule
49
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
50
+
51
+ @Test
52
+ public void testGetServiceAccountCredentialSuccess()
53
+ throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
54
+ {
55
+ GcsAuthentication auth = new GcsAuthentication(
56
+ "private_key",
57
+ GCP_EMAIL,
58
+ GCP_P12_KEYFILE,
59
+ null,
60
+ GCP_APPLICATION_NAME
61
+ );
62
+
63
+ Field field = GcsAuthentication.class.getDeclaredField("credentials");
64
+ field.setAccessible(true);
65
+
66
+ assertEquals(GoogleCredential.class, field.get(auth).getClass());
67
+ }
68
+
69
+ @Test(expected = FileNotFoundException.class)
70
+ public void testGetServiceAccountCredentialThrowFileNotFoundException()
71
+ throws GeneralSecurityException, IOException
72
+ {
73
+ Optional<String> notFoundP12Keyfile = Optional.of("/path/to/notfound.p12");
74
+ GcsAuthentication auth = new GcsAuthentication(
75
+ "private_key",
76
+ GCP_EMAIL,
77
+ notFoundP12Keyfile,
78
+ null,
79
+ GCP_APPLICATION_NAME
80
+ );
81
+ }
82
+
83
+ @Test
84
+ public void testGetGcsClientUsingServiceAccountCredentialSuccess() throws Exception
85
+ {
86
+ GcsAuthentication auth = new GcsAuthentication(
87
+ "private_key",
88
+ GCP_EMAIL,
89
+ GCP_P12_KEYFILE,
90
+ null,
91
+ GCP_APPLICATION_NAME
92
+ );
93
+
94
+ Storage client = auth.getGcsClient(GCP_BUCKET, 3);
95
+
96
+ assertEquals(Storage.class, client.getClass());
97
+ }
98
+
99
+ @Test(expected = ConfigException.class)
100
+ public void testGetGcsClientUsingServiceAccountCredentialThrowConfigException() throws Exception
101
+ {
102
+ GcsAuthentication auth = new GcsAuthentication(
103
+ "private_key",
104
+ GCP_EMAIL,
105
+ GCP_P12_KEYFILE,
106
+ null,
107
+ GCP_APPLICATION_NAME
108
+ );
109
+
110
+ Storage client = auth.getGcsClient("non-exists-bucket", 3);
111
+
112
+ assertEquals(Storage.class, client.getClass());
113
+ }
114
+
115
+ @Test
116
+ public void testGetServiceAccountCredentialFromJsonFileSuccess()
117
+ throws NoSuchFieldException, IllegalAccessException, GeneralSecurityException, IOException
118
+ {
119
+ GcsAuthentication auth = new GcsAuthentication(
120
+ "json_key",
121
+ GCP_EMAIL,
122
+ null,
123
+ GCP_JSON_KEYFILE,
124
+ GCP_APPLICATION_NAME
125
+ );
126
+ Field field = GcsAuthentication.class.getDeclaredField("credentials");
127
+ field.setAccessible(true);
128
+
129
+ assertEquals(GoogleCredential.class, field.get(auth).getClass());
130
+ }
131
+
132
+ @Test(expected = FileNotFoundException.class)
133
+ public void testGetServiceAccountCredentialFromJsonThrowFileFileNotFoundException()
134
+ throws GeneralSecurityException, IOException
135
+ {
136
+ Optional<String> notFoundJsonKeyfile = Optional.of("/path/to/notfound.json");
137
+ GcsAuthentication auth = new GcsAuthentication(
138
+ "json_key",
139
+ GCP_EMAIL,
140
+ null,
141
+ notFoundJsonKeyfile,
142
+ GCP_APPLICATION_NAME
143
+ );
144
+ }
145
+
146
+ @Test
147
+ public void testGetServiceAccountCredentialFromJsonSuccess() throws Exception
148
+ {
149
+ GcsAuthentication auth = new GcsAuthentication(
150
+ "json_key",
151
+ GCP_EMAIL,
152
+ null,
153
+ GCP_JSON_KEYFILE,
154
+ GCP_APPLICATION_NAME
155
+ );
156
+
157
+ Storage client = auth.getGcsClient(GCP_BUCKET, 3);
158
+
159
+ assertEquals(Storage.class, client.getClass());
160
+ }
161
+
162
+ @Test(expected = ConfigException.class)
163
+ public void testGetServiceAccountCredentialFromJsonThrowConfigException() throws Exception
164
+ {
165
+ GcsAuthentication auth = new GcsAuthentication(
166
+ "json_key",
167
+ GCP_EMAIL,
168
+ null,
169
+ GCP_JSON_KEYFILE,
170
+ GCP_APPLICATION_NAME
171
+ );
172
+
173
+ Storage client = auth.getGcsClient("non-exists-bucket", 3);
174
+ }
177
175
  }
data/src/test/java/org/embulk/output/TestGcsOutputPlugin.java CHANGED
@@ -1,426 +1,426 @@
1
1
  package org.embulk.output;
2
2
 
3
- import java.io.BufferedReader;
4
- import java.io.ByteArrayOutputStream;
5
- import java.io.FileInputStream;
6
- import java.io.InputStream;
7
- import java.io.InputStreamReader;
8
- import java.util.Arrays;
9
- import java.util.List;
10
-
11
- import com.google.common.collect.ImmutableMap;
3
+ import com.google.api.services.storage.Storage;
12
4
  import com.google.common.base.Optional;
13
5
  import com.google.common.collect.ImmutableList;
6
+ import com.google.common.collect.ImmutableMap;
14
7
  import com.google.common.collect.Lists;
15
- import java.io.IOException;
16
- import java.security.GeneralSecurityException;
17
8
 
18
9
  import org.embulk.EmbulkTestRuntime;
10
+ import org.embulk.config.ConfigException;
11
+ import org.embulk.config.ConfigSource;
19
12
  import org.embulk.config.TaskReport;
20
13
  import org.embulk.config.TaskSource;
21
- import org.embulk.config.ConfigDiff;
22
- import org.embulk.config.ConfigSource;
23
- import org.embulk.config.ConfigException;
14
+ import org.embulk.output.GcsOutputPlugin.PluginTask;
24
15
  import org.embulk.spi.Buffer;
25
16
  import org.embulk.spi.Exec;
26
17
  import org.embulk.spi.FileOutputPlugin;
27
18
  import org.embulk.spi.FileOutputRunner;
28
19
  import org.embulk.spi.OutputPlugin;
29
- import org.embulk.spi.TransactionalFileOutput;
30
20
  import org.embulk.spi.Schema;
31
- import org.embulk.output.GcsOutputPlugin.PluginTask;
21
+ import org.embulk.spi.TransactionalFileOutput;
32
22
  import org.embulk.standards.CsvParserPlugin;
33
23
 
34
- import org.junit.BeforeClass;
35
24
  import org.junit.Before;
25
+ import org.junit.BeforeClass;
36
26
  import org.junit.Rule;
37
27
  import org.junit.Test;
38
28
  import static org.junit.Assert.assertEquals;
39
29
  import static org.junit.Assume.assumeNotNull;
40
- import java.lang.reflect.Method;
41
- import java.lang.reflect.InvocationTargetException;
42
30
 
43
- import com.google.api.services.storage.Storage;
31
+ import java.io.BufferedReader;
32
+ import java.io.ByteArrayOutputStream;
33
+ import java.io.FileInputStream;
34
+ import java.io.IOException;
35
+ import java.io.InputStream;
36
+ import java.io.InputStreamReader;
37
+ import java.lang.reflect.InvocationTargetException;
38
+ import java.lang.reflect.Method;
39
+ import java.security.GeneralSecurityException;
40
+ import java.util.Arrays;
41
+ import java.util.List;
44
42
 
45
43
  public class TestGcsOutputPlugin
46
44
  {
47
- private static Optional<String> GCP_EMAIL;
48
- private static Optional<String> GCP_P12_KEYFILE;
49
- private static Optional<String> GCP_JSON_KEYFILE;
50
- private static String GCP_BUCKET;
51
- private static String GCP_BUCKET_DIRECTORY;
52
- private static String GCP_PATH_PREFIX;
53
- private static String LOCAL_PATH_PREFIX;
54
- private final String GCP_APPLICATION_NAME = "embulk-output-gcs";
55
- private FileOutputRunner runner;
56
-
57
- /*
58
- * This test case requires environment variables
59
- * GCP_EMAIL
60
- * GCP_P12_KEYFILE
61
- * GCP_JSON_KEYFILE
62
- * GCP_BUCKET
63
- */
64
- @BeforeClass
65
- public static void initializeConstant()
66
- {
67
- GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
68
- GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
69
- GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
70
- GCP_BUCKET = System.getenv("GCP_BUCKET");
71
- // skip test cases, if environment variables are not set.
72
- assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
73
-
74
- GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
75
- GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
76
- LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
77
- }
78
-
79
- @Rule
80
- public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
81
- private GcsOutputPlugin plugin;
82
-
83
- @Before
84
- public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
85
- {
86
- plugin = new GcsOutputPlugin();
87
- runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
88
- }
89
-
90
- @Test
91
- public void checkDefaultValues()
92
- {
93
- ConfigSource config = Exec.newConfigSource()
94
- .set("in", inputConfig())
95
- .set("parser", parserConfig(schemaConfig()))
96
- .set("type", "gcs")
97
- .set("bucket", GCP_BUCKET)
98
- .set("path_prefix", "my-prefix")
99
- .set("file_ext", ".csv")
100
- .set("formatter", formatterConfig());
101
-
102
- GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
103
- assertEquals("private_key", task.getAuthMethod().toString());
104
- }
105
-
106
- // p12_keyfile is null when auth_method is private_key
107
- @Test(expected = ConfigException.class)
108
- public void checkDefaultValuesP12keyNull()
109
- {
110
- ConfigSource config = Exec.newConfigSource()
111
- .set("in", inputConfig())
112
- .set("parser", parserConfig(schemaConfig()))
113
- .set("type", "gcs")
114
- .set("bucket", GCP_BUCKET)
115
- .set("path_prefix", "my-prefix")
116
- .set("file_ext", ".csv")
117
- .set("auth_method", "private_key")
118
- .set("service_account_email", GCP_EMAIL)
119
- .set("p12_keyfile", null)
120
- .set("formatter", formatterConfig());
121
-
122
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
123
-
124
- runner.transaction(config, schema, 0, new Control());
125
- }
126
-
127
- // both p12_keyfile and p12_keyfile_path set
128
- @Test(expected = ConfigException.class)
129
- public void checkDefaultValuesConflictSetting()
130
- {
131
- ConfigSource config = Exec.newConfigSource()
132
- .set("in", inputConfig())
133
- .set("parser", parserConfig(schemaConfig()))
134
- .set("type", "gcs")
135
- .set("bucket", GCP_BUCKET)
136
- .set("path_prefix", "my-prefix")
137
- .set("file_ext", ".csv")
138
- .set("auth_method", "private_key")
139
- .set("service_account_email", GCP_EMAIL)
140
- .set("p12_keyfile", GCP_P12_KEYFILE)
141
- .set("p12_keyfile_path", GCP_P12_KEYFILE)
142
- .set("formatter", formatterConfig());
143
-
144
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
145
-
146
- runner.transaction(config, schema, 0, new Control());
147
- }
148
-
149
- // invalid p12keyfile when auth_method is private_key
150
- @Test(expected = ConfigException.class)
151
- public void checkDefaultValuesInvalidPrivateKey()
152
- {
153
- ConfigSource config = Exec.newConfigSource()
154
- .set("in", inputConfig())
155
- .set("parser", parserConfig(schemaConfig()))
156
- .set("type", "gcs")
157
- .set("bucket", GCP_BUCKET)
158
- .set("path_prefix", "my-prefix")
159
- .set("file_ext", ".csv")
160
- .set("auth_method", "private_key")
161
- .set("service_account_email", GCP_EMAIL)
162
- .set("p12_keyfile", "invalid-key.p12")
163
- .set("formatter", formatterConfig());
164
-
165
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
166
-
167
- runner.transaction(config, schema, 0, new Control());
168
- }
169
-
170
- // json_keyfile is null when auth_method is json_key
171
- @Test(expected = ConfigException.class)
172
- public void checkDefaultValuesJsonKeyfileNull()
173
- {
174
- ConfigSource config = Exec.newConfigSource()
175
- .set("in", inputConfig())
176
- .set("parser", parserConfig(schemaConfig()))
177
- .set("type", "gcs")
178
- .set("bucket", GCP_BUCKET)
179
- .set("path_prefix", "my-prefix")
180
- .set("file_ext", ".csv")
181
- .set("auth_method", "json_key")
182
- .set("service_account_email", GCP_EMAIL)
183
- .set("json_keyfile", null)
184
- .set("formatter", formatterConfig());
185
-
186
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
187
-
188
- runner.transaction(config, schema, 0, new Control());
189
- }
190
-
191
- @Test
192
- public void testGcsClientCreateSuccessfully()
193
- throws GeneralSecurityException, IOException, NoSuchMethodException,
194
- IllegalAccessException, InvocationTargetException
195
- {
196
- ConfigSource configSource = config();
197
- PluginTask task = configSource.loadConfig(PluginTask.class);
198
- Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
199
- runner.transaction(configSource, schema, 0, new Control());
200
-
201
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
202
- method.setAccessible(true);
203
- method.invoke(plugin, task); // no errors happens
204
- }
205
-
206
- @Test(expected = ConfigException.class)
207
- public void testGcsClientCreateThrowConfigException()
208
- throws GeneralSecurityException, IOException, NoSuchMethodException,
209
- IllegalAccessException, InvocationTargetException
210
- {
211
- ConfigSource config = Exec.newConfigSource()
212
- .set("in", inputConfig())
213
- .set("parser", parserConfig(schemaConfig()))
214
- .set("type", "gcs")
215
- .set("bucket", "non-exists-bucket")
216
- .set("path_prefix", "my-prefix")
217
- .set("file_ext", ".csv")
218
- .set("auth_method", "json_key")
219
- .set("service_account_email", GCP_EMAIL)
220
- .set("json_keyfile", GCP_JSON_KEYFILE)
221
- .set("formatter", formatterConfig());
222
-
223
- PluginTask task = config.loadConfig(PluginTask.class);
224
-
225
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
226
- runner.transaction(config, schema, 0, new Control());
227
-
228
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
229
- method.setAccessible(true);
230
- try {
231
- method.invoke(plugin, task);
232
- } catch (InvocationTargetException ex) {
233
- throw (ConfigException) ex.getCause();
234
- }
235
- }
236
-
237
- @Test
238
- public void testResume()
239
- {
240
- PluginTask task = config().loadConfig(PluginTask.class);
241
- plugin.resume(task.dump(), 0, new FileOutputPlugin.Control() // no errors happens
242
- {
243
- @Override
244
- public List<TaskReport> run(TaskSource taskSource)
245
- {
246
- return Lists.newArrayList(Exec.newTaskReport());
247
- }
248
- });
249
- }
250
-
251
- @Test
252
- public void testCleanup()
253
- {
254
- PluginTask task = config().loadConfig(PluginTask.class);
255
- plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
256
- }
257
-
258
- @Test
259
- public void testGcsFileOutputByOpen() throws Exception
260
- {
261
- ConfigSource configSource = config();
262
- PluginTask task = configSource.loadConfig(PluginTask.class);
263
- Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
264
- runner.transaction(configSource, schema, 0, new Control());
265
-
266
- TransactionalFileOutput output = plugin.open(task.dump(), 0);
267
-
268
- output.nextFile();
269
-
270
- FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
271
- byte[] bytes = convertInputStreamToByte(is);
272
- Buffer buffer = Buffer.wrap(bytes);
273
- output.add(buffer);
274
-
275
- output.finish();
276
- output.commit();
277
-
278
- String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
279
- assertRecords(remotePath);
280
- }
281
-
282
- public ConfigSource config()
283
- {
284
- return Exec.newConfigSource()
285
- .set("in", inputConfig())
286
- .set("parser", parserConfig(schemaConfig()))
287
- .set("type", "gcs")
288
- .set("bucket", GCP_BUCKET)
289
- .set("path_prefix", GCP_PATH_PREFIX)
290
- .set("last_path", "")
291
- .set("file_ext", ".csv")
292
- .set("auth_method", "private_key")
293
- .set("service_account_email", GCP_EMAIL)
294
- .set("p12_keyfile", GCP_P12_KEYFILE)
295
- .set("json_keyfile", GCP_JSON_KEYFILE)
296
- .set("application_name", GCP_APPLICATION_NAME)
297
- .set("formatter", formatterConfig());
298
- }
299
-
300
- private class Control
301
- implements OutputPlugin.Control
302
- {
303
- @Override
304
- public List<TaskReport> run(TaskSource taskSource)
305
- {
306
- return Lists.newArrayList(Exec.newTaskReport());
307
- }
308
- }
309
-
310
- private ImmutableMap<String, Object> inputConfig()
311
- {
312
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
313
- builder.put("type", "file");
314
- builder.put("path_prefix", LOCAL_PATH_PREFIX);
315
- builder.put("last_path", "");
316
- return builder.build();
317
- }
318
-
319
- private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
320
- {
321
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
322
- builder.put("type", "csv");
323
- builder.put("newline", "CRLF");
324
- builder.put("delimiter", ",");
325
- builder.put("quote", "\"");
326
- builder.put("escape", "\"");
327
- builder.put("trim_if_not_quoted", false);
328
- builder.put("skip_header_lines", 1);
329
- builder.put("allow_extra_columns", false);
330
- builder.put("allow_optional_columns", false);
331
- builder.put("columns", schemaConfig);
332
- return builder.build();
333
- }
334
-
335
- private ImmutableList<Object> schemaConfig()
336
- {
337
- ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
338
- builder.add(ImmutableMap.of("name", "id", "type", "long"));
339
- builder.add(ImmutableMap.of("name", "account", "type", "long"));
340
- builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
341
- builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
342
- builder.add(ImmutableMap.of("name", "comment", "type", "string"));
343
- return builder.build();
344
- }
345
-
346
- private ImmutableMap<String, Object> formatterConfig()
347
- {
348
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
349
- builder.put("type", "csv");
350
- builder.put("header_line", "false");
351
- builder.put("timezone", "Asia/Tokyo");
352
- return builder.build();
353
- }
354
-
355
- private void assertRecords(String gcsPath) throws Exception
356
- {
357
- ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
358
- assertEquals(5, records.size());
359
- {
360
- List<String> record = records.get(1);
361
- assertEquals("1", record.get(0));
362
- assertEquals("32864", record.get(1));
363
- assertEquals("2015-01-27 19:23:49", record.get(2));
364
- assertEquals("20150127", record.get(3));
365
- assertEquals("embulk", record.get(4));
366
- }
367
-
368
- {
369
- List<String> record = records.get(2);
370
- assertEquals("2", record.get(0));
371
- assertEquals("14824", record.get(1));
372
- assertEquals("2015-01-27 19:01:23", record.get(2));
373
- assertEquals("20150127", record.get(3));
374
- assertEquals("embulk jruby", record.get(4));
375
- }
376
- }
377
-
378
- private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
379
- {
380
- ConfigSource config = config();
381
-
382
- PluginTask task = config.loadConfig(PluginTask.class);
383
-
384
- Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
385
- method.setAccessible(true);
386
- Storage client = (Storage) method.invoke(plugin, task);
387
- Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
388
-
389
- ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
390
-
391
- InputStream is = getObject.executeMediaAsInputStream();
392
- BufferedReader reader = new BufferedReader(new InputStreamReader(is));
393
- String line;
394
- while ((line = reader.readLine()) != null) {
395
- List<String> records = Arrays.asList(line.split(",", 0));
396
-
397
- builder.add(records);
398
- }
399
- return builder.build();
400
- }
401
-
402
- private static String getDirectory(String dir)
403
- {
404
- if (dir != null && !dir.endsWith("/")) {
405
- dir = dir + "/";
406
- }
407
- if (dir.startsWith("/")) {
408
- dir = dir.replaceFirst("/", "");
409
- }
410
- return dir;
411
- }
412
-
413
- private byte[] convertInputStreamToByte(InputStream is) throws IOException
414
- {
415
- ByteArrayOutputStream bo = new ByteArrayOutputStream();
416
- byte [] buffer = new byte[1024];
417
- while(true) {
418
- int len = is.read(buffer);
419
- if(len < 0) {
420
- break;
421
- }
422
- bo.write(buffer, 0, len);
423
- }
424
- return bo.toByteArray();
425
- }
45
+ private static Optional<String> GCP_EMAIL;
46
+ private static Optional<String> GCP_P12_KEYFILE;
47
+ private static Optional<String> GCP_JSON_KEYFILE;
48
+ private static String GCP_BUCKET;
49
+ private static String GCP_BUCKET_DIRECTORY;
50
+ private static String GCP_PATH_PREFIX;
51
+ private static String LOCAL_PATH_PREFIX;
52
+ private static String GCP_APPLICATION_NAME;
53
+ private FileOutputRunner runner;
54
+
55
+ /*
56
+ * This test case requires environment variables
57
+ * GCP_EMAIL
58
+ * GCP_P12_KEYFILE
59
+ * GCP_JSON_KEYFILE
60
+ * GCP_BUCKET
61
+ */
62
+ @BeforeClass
63
+ public static void initializeConstant()
64
+ {
65
+ GCP_EMAIL = Optional.of(System.getenv("GCP_EMAIL"));
66
+ GCP_P12_KEYFILE = Optional.of(System.getenv("GCP_P12_KEYFILE"));
67
+ GCP_JSON_KEYFILE = Optional.of(System.getenv("GCP_JSON_KEYFILE"));
68
+ GCP_BUCKET = System.getenv("GCP_BUCKET");
69
+ // skip test cases, if environment variables are not set.
70
+ assumeNotNull(GCP_EMAIL, GCP_P12_KEYFILE, GCP_JSON_KEYFILE, GCP_BUCKET);
71
+
72
+ GCP_BUCKET_DIRECTORY = System.getenv("GCP_BUCKET_DIRECTORY") != null ? getDirectory(System.getenv("GCP_BUCKET_DIRECTORY")) : getDirectory("");
73
+ GCP_PATH_PREFIX = GCP_BUCKET_DIRECTORY + "sample_";
74
+ LOCAL_PATH_PREFIX = GcsOutputPlugin.class.getClassLoader().getResource("sample_01.csv").getPath();
75
+ GCP_APPLICATION_NAME = "embulk-output-gcs";
76
+ }
77
+
78
+ @Rule
79
+ public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
80
+ private GcsOutputPlugin plugin;
81
+
82
+ @Before
83
+ public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
84
+ {
85
+ plugin = new GcsOutputPlugin();
86
+ runner = new FileOutputRunner(runtime.getInstance(GcsOutputPlugin.class));
87
+ }
88
+
89
+ @Test
90
+ public void checkDefaultValues()
91
+ {
92
+ ConfigSource config = Exec.newConfigSource()
93
+ .set("in", inputConfig())
94
+ .set("parser", parserConfig(schemaConfig()))
95
+ .set("type", "gcs")
96
+ .set("bucket", GCP_BUCKET)
97
+ .set("path_prefix", "my-prefix")
98
+ .set("file_ext", ".csv")
99
+ .set("formatter", formatterConfig());
100
+
101
+ GcsOutputPlugin.PluginTask task = config.loadConfig(PluginTask.class);
102
+ assertEquals("private_key", task.getAuthMethod().toString());
103
+ }
104
+
105
+ // p12_keyfile is null when auth_method is private_key
106
+ @Test(expected = ConfigException.class)
107
+ public void checkDefaultValuesP12keyNull()
108
+ {
109
+ ConfigSource config = Exec.newConfigSource()
110
+ .set("in", inputConfig())
111
+ .set("parser", parserConfig(schemaConfig()))
112
+ .set("type", "gcs")
113
+ .set("bucket", GCP_BUCKET)
114
+ .set("path_prefix", "my-prefix")
115
+ .set("file_ext", ".csv")
116
+ .set("auth_method", "private_key")
117
+ .set("service_account_email", GCP_EMAIL)
118
+ .set("p12_keyfile", null)
119
+ .set("formatter", formatterConfig());
120
+
121
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
122
+
123
+ runner.transaction(config, schema, 0, new Control());
124
+ }
125
+
126
+ // both p12_keyfile and p12_keyfile_path set
127
+ @Test(expected = ConfigException.class)
128
+ public void checkDefaultValuesConflictSetting()
129
+ {
130
+ ConfigSource config = Exec.newConfigSource()
131
+ .set("in", inputConfig())
132
+ .set("parser", parserConfig(schemaConfig()))
133
+ .set("type", "gcs")
134
+ .set("bucket", GCP_BUCKET)
135
+ .set("path_prefix", "my-prefix")
136
+ .set("file_ext", ".csv")
137
+ .set("auth_method", "private_key")
138
+ .set("service_account_email", GCP_EMAIL)
139
+ .set("p12_keyfile", GCP_P12_KEYFILE)
140
+ .set("p12_keyfile_path", GCP_P12_KEYFILE)
141
+ .set("formatter", formatterConfig());
142
+
143
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
144
+
145
+ runner.transaction(config, schema, 0, new Control());
146
+ }
147
+
148
+ // invalid p12keyfile when auth_method is private_key
149
+ @Test(expected = ConfigException.class)
150
+ public void checkDefaultValuesInvalidPrivateKey()
151
+ {
152
+ ConfigSource config = Exec.newConfigSource()
153
+ .set("in", inputConfig())
154
+ .set("parser", parserConfig(schemaConfig()))
155
+ .set("type", "gcs")
156
+ .set("bucket", GCP_BUCKET)
157
+ .set("path_prefix", "my-prefix")
158
+ .set("file_ext", ".csv")
159
+ .set("auth_method", "private_key")
160
+ .set("service_account_email", GCP_EMAIL)
161
+ .set("p12_keyfile", "invalid-key.p12")
162
+ .set("formatter", formatterConfig());
163
+
164
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
165
+
166
+ runner.transaction(config, schema, 0, new Control());
167
+ }
168
+
169
+ // json_keyfile is null when auth_method is json_key
170
+ @Test(expected = ConfigException.class)
171
+ public void checkDefaultValuesJsonKeyfileNull()
172
+ {
173
+ ConfigSource config = Exec.newConfigSource()
174
+ .set("in", inputConfig())
175
+ .set("parser", parserConfig(schemaConfig()))
176
+ .set("type", "gcs")
177
+ .set("bucket", GCP_BUCKET)
178
+ .set("path_prefix", "my-prefix")
179
+ .set("file_ext", ".csv")
180
+ .set("auth_method", "json_key")
181
+ .set("service_account_email", GCP_EMAIL)
182
+ .set("json_keyfile", null)
183
+ .set("formatter", formatterConfig());
184
+
185
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
186
+
187
+ runner.transaction(config, schema, 0, new Control());
188
+ }
189
+
190
+ @Test
191
+ public void testGcsClientCreateSuccessfully()
192
+ throws GeneralSecurityException, IOException, NoSuchMethodException,
193
+ IllegalAccessException, InvocationTargetException
194
+ {
195
+ ConfigSource configSource = config();
196
+ PluginTask task = configSource.loadConfig(PluginTask.class);
197
+ Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
198
+ runner.transaction(configSource, schema, 0, new Control());
199
+
200
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
201
+ method.setAccessible(true);
202
+ method.invoke(plugin, task); // no errors happens
203
+ }
204
+
205
+ @Test(expected = ConfigException.class)
206
+ public void testGcsClientCreateThrowConfigException()
207
+ throws GeneralSecurityException, IOException, NoSuchMethodException,
208
+ IllegalAccessException, InvocationTargetException
209
+ {
210
+ ConfigSource config = Exec.newConfigSource()
211
+ .set("in", inputConfig())
212
+ .set("parser", parserConfig(schemaConfig()))
213
+ .set("type", "gcs")
214
+ .set("bucket", "non-exists-bucket")
215
+ .set("path_prefix", "my-prefix")
216
+ .set("file_ext", ".csv")
217
+ .set("auth_method", "json_key")
218
+ .set("service_account_email", GCP_EMAIL)
219
+ .set("json_keyfile", GCP_JSON_KEYFILE)
220
+ .set("formatter", formatterConfig());
221
+
222
+ PluginTask task = config.loadConfig(PluginTask.class);
223
+
224
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
225
+ runner.transaction(config, schema, 0, new Control());
226
+
227
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
228
+ method.setAccessible(true);
229
+ try {
230
+ method.invoke(plugin, task);
231
+ }
232
+ catch (InvocationTargetException ex) {
233
+ throw (ConfigException) ex.getCause();
234
+ }
235
+ }
236
+
237
+ @Test
238
+ public void testResume()
239
+ {
240
+ PluginTask task = config().loadConfig(PluginTask.class);
241
+ plugin.resume(task.dump(), 0, new FileOutputPlugin.Control() // no errors happens
242
+ {
243
+ @Override
244
+ public List<TaskReport> run(TaskSource taskSource)
245
+ {
246
+ return Lists.newArrayList(Exec.newTaskReport());
247
+ }
248
+ });
249
+ }
250
+
251
+ @Test
252
+ public void testCleanup()
253
+ {
254
+ PluginTask task = config().loadConfig(PluginTask.class);
255
+ plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
256
+ }
257
+
258
+ @Test
259
+ public void testGcsFileOutputByOpen() throws Exception
260
+ {
261
+ ConfigSource configSource = config();
262
+ PluginTask task = configSource.loadConfig(PluginTask.class);
263
+ Schema schema = configSource.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
264
+ runner.transaction(configSource, schema, 0, new Control());
265
+
266
+ TransactionalFileOutput output = plugin.open(task.dump(), 0);
267
+
268
+ output.nextFile();
269
+
270
+ FileInputStream is = new FileInputStream(LOCAL_PATH_PREFIX);
271
+ byte[] bytes = convertInputStreamToByte(is);
272
+ Buffer buffer = Buffer.wrap(bytes);
273
+ output.add(buffer);
274
+
275
+ output.finish();
276
+ output.commit();
277
+
278
+ String remotePath = GCP_PATH_PREFIX + String.format(task.getSequenceFormat(), 0, 0) + task.getFileNameExtension();
279
+ assertRecords(remotePath);
280
+ }
281
+
282
+ public ConfigSource config()
283
+ {
284
+ return Exec.newConfigSource()
285
+ .set("in", inputConfig())
286
+ .set("parser", parserConfig(schemaConfig()))
287
+ .set("type", "gcs")
288
+ .set("bucket", GCP_BUCKET)
289
+ .set("path_prefix", GCP_PATH_PREFIX)
290
+ .set("last_path", "")
291
+ .set("file_ext", ".csv")
292
+ .set("auth_method", "private_key")
293
+ .set("service_account_email", GCP_EMAIL)
294
+ .set("p12_keyfile", GCP_P12_KEYFILE)
295
+ .set("json_keyfile", GCP_JSON_KEYFILE)
296
+ .set("application_name", GCP_APPLICATION_NAME)
297
+ .set("formatter", formatterConfig());
298
+ }
299
+
300
+ private class Control
301
+ implements OutputPlugin.Control
302
+ {
303
+ @Override
304
+ public List<TaskReport> run(TaskSource taskSource)
305
+ {
306
+ return Lists.newArrayList(Exec.newTaskReport());
307
+ }
308
+ }
309
+
310
+ private ImmutableMap<String, Object> inputConfig()
311
+ {
312
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
313
+ builder.put("type", "file");
314
+ builder.put("path_prefix", LOCAL_PATH_PREFIX);
315
+ builder.put("last_path", "");
316
+ return builder.build();
317
+ }
318
+
319
+ private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
320
+ {
321
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
322
+ builder.put("type", "csv");
323
+ builder.put("newline", "CRLF");
324
+ builder.put("delimiter", ",");
325
+ builder.put("quote", "\"");
326
+ builder.put("escape", "\"");
327
+ builder.put("trim_if_not_quoted", false);
328
+ builder.put("skip_header_lines", 1);
329
+ builder.put("allow_extra_columns", false);
330
+ builder.put("allow_optional_columns", false);
331
+ builder.put("columns", schemaConfig);
332
+ return builder.build();
333
+ }
334
+
335
+ private ImmutableList<Object> schemaConfig()
336
+ {
337
+ ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
338
+ builder.add(ImmutableMap.of("name", "id", "type", "long"));
339
+ builder.add(ImmutableMap.of("name", "account", "type", "long"));
340
+ builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
341
+ builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
342
+ builder.add(ImmutableMap.of("name", "comment", "type", "string"));
343
+ return builder.build();
344
+ }
345
+
346
+ private ImmutableMap<String, Object> formatterConfig()
347
+ {
348
+ ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
349
+ builder.put("type", "csv");
350
+ builder.put("header_line", "false");
351
+ builder.put("timezone", "Asia/Tokyo");
352
+ return builder.build();
353
+ }
354
+
355
+ private void assertRecords(String gcsPath) throws Exception
356
+ {
357
+ ImmutableList<List<String>> records = getFileContentsFromGcs(gcsPath);
358
+ assertEquals(5, records.size());
359
+ {
360
+ List<String> record = records.get(1);
361
+ assertEquals("1", record.get(0));
362
+ assertEquals("32864", record.get(1));
363
+ assertEquals("2015-01-27 19:23:49", record.get(2));
364
+ assertEquals("20150127", record.get(3));
365
+ assertEquals("embulk", record.get(4));
366
+ }
367
+
368
+ {
369
+ List<String> record = records.get(2);
370
+ assertEquals("2", record.get(0));
371
+ assertEquals("14824", record.get(1));
372
+ assertEquals("2015-01-27 19:01:23", record.get(2));
373
+ assertEquals("20150127", record.get(3));
374
+ assertEquals("embulk jruby", record.get(4));
375
+ }
376
+ }
377
+
378
+ private ImmutableList<List<String>> getFileContentsFromGcs(String path) throws Exception
379
+ {
380
+ ConfigSource config = config();
381
+
382
+ PluginTask task = config.loadConfig(PluginTask.class);
383
+
384
+ Method method = GcsOutputPlugin.class.getDeclaredMethod("createClient", PluginTask.class);
385
+ method.setAccessible(true);
386
+ Storage client = (Storage) method.invoke(plugin, task);
387
+ Storage.Objects.Get getObject = client.objects().get(GCP_BUCKET, path);
388
+
389
+ ImmutableList.Builder<List<String>> builder = new ImmutableList.Builder<>();
390
+
391
+ InputStream is = getObject.executeMediaAsInputStream();
392
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is));
393
+ String line;
394
+ while ((line = reader.readLine()) != null) {
395
+ List<String> records = Arrays.asList(line.split(",", 0));
396
+
397
+ builder.add(records);
398
+ }
399
+ return builder.build();
400
+ }
401
+
402
+ private static String getDirectory(String dir)
403
+ {
404
+ if (dir != null && !dir.endsWith("/")) {
405
+ dir = dir + "/";
406
+ }
407
+ if (dir.startsWith("/")) {
408
+ dir = dir.replaceFirst("/", "");
409
+ }
410
+ return dir;
411
+ }
412
+
413
+ private byte[] convertInputStreamToByte(InputStream is) throws IOException
414
+ {
415
+ ByteArrayOutputStream bo = new ByteArrayOutputStream();
416
+ byte [] buffer = new byte[1024];
417
+ while (true) {
418
+ int len = is.read(buffer);
419
+ if (len < 0) {
420
+ break;
421
+ }
422
+ bo.write(buffer, 0, len);
423
+ }
424
+ return bo.toByteArray();
425
+ }
426
426
  }