embulk-output-oracle 0.4.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +160 -160
  3. data/build.gradle +6 -6
  4. data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
  5. data/classpath/{embulk-output-oracle-0.4.1.jar → embulk-output-oracle-0.4.2.jar} +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -153
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +179 -165
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +185 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +50 -25
  37. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +548 -540
  38. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  39. data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java +79 -0
  40. data/src/test/resources/data/test1/test1.csv +3 -3
  41. data/src/test/resources/dummy.txt +1 -0
  42. data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -26
  43. data/src/test/resources/yml/test-insert-direct-empty.yml +25 -25
  44. data/src/test/resources/yml/test-insert-direct-oci-method-split.yml +28 -28
  45. data/src/test/resources/yml/test-insert-direct-oci-method.yml +28 -28
  46. data/src/test/resources/yml/test-insert-direct.yml +25 -25
  47. data/src/test/resources/yml/test-insert-empty.yml +27 -27
  48. data/src/test/resources/yml/test-insert.yml +25 -27
  49. data/src/test/resources/yml/test-replace-empty.yml +31 -31
  50. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
  51. data/src/test/resources/yml/test-replace-long-name.yml +31 -31
  52. data/src/test/resources/yml/test-replace.yml +31 -31
  53. data/src/test/resources/yml/test-string-timestamp.yml +28 -28
  54. data/src/test/resources/yml/test-url.yml +24 -24
  55. metadata +6 -8
  56. data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
  57. data/lib/embulk/linux_x64/libembulk-output-oracle.so +0 -0
  58. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +0 -42
  59. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +0 -124
  60. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +0 -105
@@ -1,540 +1,548 @@
1
- package org.embulk.output.oracle;
2
-
3
- import static org.junit.Assert.assertEquals;
4
-
5
- import java.io.BufferedReader;
6
- import java.io.BufferedWriter;
7
- import java.io.File;
8
- import java.io.FileReader;
9
- import java.io.FileWriter;
10
- import java.io.IOException;
11
- import java.lang.reflect.Constructor;
12
- import java.math.BigDecimal;
13
- import java.net.URISyntaxException;
14
- import java.sql.Connection;
15
- import java.sql.DriverManager;
16
- import java.sql.ResultSet;
17
- import java.sql.SQLException;
18
- import java.sql.Statement;
19
- import java.sql.Timestamp;
20
- import java.text.DateFormat;
21
- import java.text.ParseException;
22
- import java.text.SimpleDateFormat;
23
- import java.util.ArrayList;
24
- import java.util.Collections;
25
- import java.util.Comparator;
26
- import java.util.Date;
27
- import java.util.Iterator;
28
- import java.util.List;
29
- import java.util.TimeZone;
30
- import java.util.regex.Matcher;
31
- import java.util.regex.Pattern;
32
-
33
- import org.embulk.input.filesplit.LocalFileSplitInputPlugin;
34
- import org.embulk.output.OracleOutputPlugin;
35
- import org.embulk.spi.InputPlugin;
36
- import org.embulk.spi.OutputPlugin;
37
-
38
-
39
- public class OracleOutputPluginTestImpl
40
- {
41
- private EmbulkPluginTester tester = new EmbulkPluginTester(OutputPlugin.class, "oracle", OracleOutputPlugin.class);
42
-
43
-
44
- public String beforeClass()
45
- {
46
- try {
47
- Class.forName("oracle.jdbc.OracleDriver");
48
-
49
- try (Connection connection = connect()) {
50
- String version = connection.getMetaData().getDriverVersion();
51
- System.out.println("Driver version = " + version);
52
- return version;
53
- }
54
-
55
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
56
- //throw new RuntimeException("You should put Oracle JDBC driver on 'driver' directory.");
57
- System.err.println("Warning: put Oracle JDBC driver on 'driver' directory in order to test embulk-output-oracle plugin.");
58
-
59
- } catch (SQLException e) {
60
- System.err.println(e);
61
- //throw new RuntimeException("You should prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
62
- System.err.println("Warning: prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
63
- // for example
64
- // CREATE USER EMBULK_USER IDENTIFIED BY "embulk_pw";
65
- // GRANT DBA TO EMBULK_USER;
66
- }
67
-
68
- convertPath("/data/test2/").mkdirs();
69
-
70
- return null;
71
- }
72
-
73
- public void testInsert() throws Exception
74
- {
75
- String table = "TEST1";
76
-
77
- dropTable(table);
78
- createTable(table);
79
-
80
- run("/yml/test-insert.yml");
81
-
82
- assertTable(table);
83
- }
84
-
85
- public void testInsertEmpty() throws Exception
86
- {
87
- String table = "TEST1";
88
-
89
- dropTable(table);
90
- createTable(table);
91
-
92
- run("/yml/test-insert-empty.yml");
93
-
94
- assertTableEmpty(table);
95
- }
96
-
97
- public void testInsertDirect() throws Exception
98
- {
99
- String table = "TEST1";
100
-
101
- dropTable(table);
102
- createTable(table);
103
-
104
- run("/yml/test-insert-direct.yml");
105
-
106
- assertTable(table);
107
- }
108
-
109
- public void testInsertDirectEmpty() throws Exception
110
- {
111
- String table = "TEST1";
112
-
113
- dropTable(table);
114
- createTable(table);
115
-
116
- run("/yml/test-insert-direct-empty.yml");
117
-
118
- assertTableEmpty(table);
119
- }
120
-
121
- public void testInsertDirectCreate() throws Exception
122
- {
123
- String table = "TEST1";
124
-
125
- dropTable(table);
126
-
127
- run("/yml/test-insert-direct.yml");
128
-
129
- assertGeneratedTable1(table);
130
- }
131
-
132
- public void testInsertDirectDirectMethod() throws Exception
133
- {
134
- String table = "TEST1";
135
-
136
- dropTable(table);
137
- createTable(table);
138
-
139
- run("/yml/test-insert-direct-direct-method.yml");
140
-
141
- assertTable(table);
142
- }
143
-
144
- public void testInsertDirectOCIMethod() throws Exception
145
- {
146
- String table = "TEST1";
147
-
148
- dropTable(table);
149
- createTable(table);
150
-
151
- run("/yml/test-insert-direct-oci-method.yml");
152
-
153
- assertTable(table);
154
- }
155
-
156
- public void testInsertDirectOCIMethodSplit() throws Exception
157
- {
158
- tester.addPlugin(InputPlugin.class, "filesplit", LocalFileSplitInputPlugin.class);
159
-
160
- String table = "TEST1";
161
-
162
- dropTable(table);
163
- createTable(table);
164
-
165
- run("/yml/test-insert-direct-oci-method-split.yml");
166
-
167
- assertTable(table);
168
- }
169
-
170
- public void testUrl() throws Exception
171
- {
172
- String table = "TEST1";
173
-
174
- dropTable(table);
175
- createTable(table);
176
-
177
- run("/yml/test-url.yml");
178
-
179
- assertTable(table);
180
- }
181
-
182
- public void testReplace() throws Exception
183
- {
184
- String table = "TEST1";
185
-
186
- dropTable(table);
187
- createTable(table);
188
-
189
- run("/yml/test-replace.yml");
190
-
191
- assertGeneratedTable2(table);
192
- }
193
-
194
- public void testReplaceEmpty() throws Exception
195
- {
196
- String table = "TEST1";
197
-
198
- dropTable(table);
199
- createTable(table);
200
-
201
- run("/yml/test-replace-empty.yml");
202
-
203
- assertTableEmpty(table);
204
- }
205
-
206
- public void testReplaceCreate() throws Exception
207
- {
208
- String table = "TEST1";
209
-
210
- dropTable(table);
211
-
212
- run("/yml/test-replace.yml");
213
-
214
- assertGeneratedTable2(table);
215
- }
216
-
217
-
218
- public void testReplaceLongName() throws Exception
219
- {
220
- String table = "TEST12345678901234567890123456";
221
-
222
- dropTable(table);
223
- createTable(table);
224
-
225
- run("/yml/test-replace-long-name.yml");
226
-
227
- assertGeneratedTable2(table);
228
- }
229
-
230
- public void testReplaceLongNameMultibyte() throws Exception
231
- {
232
- String table = "TEST12345678901234567890";
233
-
234
- run("/yml/test-replace-long-name-multibyte.yml");
235
-
236
- assertGeneratedTable2(table);
237
- }
238
-
239
- public void testStringTimestamp() throws Exception
240
- {
241
- String table = "TEST1";
242
-
243
- dropTable(table);
244
- createTable(table);
245
-
246
- run("/yml/test-string-timestamp.yml");
247
-
248
- assertTable(table);
249
- }
250
-
251
- private void dropTable(String table) throws SQLException
252
- {
253
- String sql = String.format("DROP TABLE %s", table);
254
- executeSQL(sql, true);
255
- }
256
-
257
- private void createTable(String table) throws SQLException
258
- {
259
- String sql = String.format("CREATE TABLE %s ("
260
- + "ID CHAR(4),"
261
- + "VARCHAR2_ITEM VARCHAR2(40),"
262
- + "INTEGER_ITEM NUMBER(4,0),"
263
- + "NUMBER_ITEM NUMBER(10,2),"
264
- + "DATE_ITEM DATE,"
265
- + "TIMESTAMP_ITEM TIMESTAMP,"
266
- + "PRIMARY KEY (ID))", table);
267
- executeSQL(sql);
268
- }
269
-
270
- private void assertTable(String table) throws Exception
271
- {
272
- // datetime of UTC will be inserted by embulk.
273
- // datetime of default timezone will be selected by JDBC.
274
- TimeZone timeZone = TimeZone.getDefault();
275
- List<List<Object>> rows = select(table);
276
-
277
- /*
278
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
279
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
280
- A003,,,,,
281
- */
282
-
283
- assertEquals(3, rows.size());
284
- Iterator<List<Object>> i1 = rows.iterator();
285
- {
286
- Iterator<Object> i2 = i1.next().iterator();
287
- assertEquals("A001", i2.next());
288
- assertEquals("ABCDE", i2.next());
289
- assertEquals(new BigDecimal("0"), i2.next());
290
- assertEquals(new BigDecimal("123.45"), i2.next());
291
- assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
292
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
293
- }
294
- {
295
- Iterator<Object> i2 = i1.next().iterator();
296
- assertEquals("A002", i2.next());
297
- assertEquals("あいうえお", i2.next());
298
- assertEquals(new BigDecimal("-9999"), i2.next());
299
- assertEquals(new BigDecimal("-99999999.99"), i2.next());
300
- assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
301
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
302
- }
303
- {
304
- Iterator<Object> i2 = i1.next().iterator();
305
- assertEquals("A003", i2.next());
306
- assertEquals(null, i2.next());
307
- assertEquals(null, i2.next());
308
- assertEquals(null, i2.next());
309
- assertEquals(null, i2.next());
310
- assertEquals(null, i2.next());
311
- }
312
- }
313
-
314
- private void assertTableEmpty(String table) throws Exception
315
- {
316
- List<List<Object>> rows = select(table);
317
- assertEquals(0, rows.size());
318
- }
319
-
320
- private void assertGeneratedTable1(String table) throws Exception
321
- {
322
- // datetime of UTC will be inserted by embulk.
323
- // datetime of default timezone will be selected by JDBC.
324
- TimeZone timeZone = TimeZone.getDefault();
325
- List<List<Object>> rows = select(table);
326
-
327
- /*
328
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
329
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
330
- A003,,,,,
331
- */
332
-
333
- assertEquals(3, rows.size());
334
- Iterator<List<Object>> i1 = rows.iterator();
335
- {
336
- Iterator<Object> i2 = i1.next().iterator();
337
- assertEquals("A001", i2.next());
338
- assertEquals("ABCDE", i2.next());
339
- assertEquals(new BigDecimal("0"), i2.next());
340
- assertEquals("123.45", i2.next());
341
- assertEquals(toOracleTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
342
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
343
- }
344
- {
345
- Iterator<Object> i2 = i1.next().iterator();
346
- assertEquals("A002", i2.next());
347
- assertEquals("あいうえお", i2.next());
348
- assertEquals(new BigDecimal("-9999"), i2.next());
349
- assertEquals("-99999999.99", i2.next());
350
- assertEquals(toOracleTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
351
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
352
- }
353
- {
354
- Iterator<Object> i2 = i1.next().iterator();
355
- assertEquals("A003", i2.next());
356
- assertEquals(null, i2.next());
357
- assertEquals(null, i2.next());
358
- assertEquals(null, i2.next());
359
- assertEquals(null, i2.next());
360
- assertEquals(null, i2.next());
361
- }
362
- }
363
-
364
- private void assertGeneratedTable2(String table) throws Exception
365
- {
366
- // datetime of UTC will be inserted by embulk.
367
- // datetime of default timezone will be selected by JDBC.
368
- TimeZone timeZone = TimeZone.getDefault();
369
- List<List<Object>> rows = select(table);
370
-
371
- /*
372
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
373
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
374
- A003,,,,,
375
- */
376
-
377
- assertEquals(3, rows.size());
378
- Iterator<List<Object>> i1 = rows.iterator();
379
- {
380
- Iterator<Object> i2 = i1.next().iterator();
381
- assertEquals("A001", i2.next());
382
- assertEquals("ABCDE", i2.next());
383
- assertEquals(new BigDecimal("0"), i2.next());
384
- assertEquals(new BigDecimal("123.45"), i2.next());
385
- assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
386
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
387
- }
388
- {
389
- Iterator<Object> i2 = i1.next().iterator();
390
- assertEquals("A002", i2.next());
391
- assertEquals("あいうえお", i2.next());
392
- assertEquals(new BigDecimal("-9999"), i2.next());
393
- assertEquals(new BigDecimal("-99999999.99"), i2.next());
394
- assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
395
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
396
- }
397
- {
398
- Iterator<Object> i2 = i1.next().iterator();
399
- assertEquals("A003", i2.next());
400
- assertEquals(null, i2.next());
401
- assertEquals(null, i2.next());
402
- assertEquals(null, i2.next());
403
- assertEquals(null, i2.next());
404
- assertEquals(null, i2.next());
405
- }
406
- }
407
-
408
-
409
- private Timestamp toTimestamp(String s, TimeZone timeZone)
410
- {
411
- for (String formatString : new String[]{"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd"}) {
412
- DateFormat dateFormat = new SimpleDateFormat(formatString);
413
- dateFormat.setTimeZone(timeZone);
414
- try {
415
- Date date = dateFormat.parse(s);
416
- return new Timestamp(date.getTime());
417
- } catch (ParseException e) {
418
- // NOP
419
- }
420
- }
421
- throw new IllegalArgumentException(s);
422
- }
423
-
424
- private Object toOracleTimestamp(String s, TimeZone timeZone) throws Exception
425
- {
426
- Class<?> timestampClass = Class.forName("oracle.sql.TIMESTAMP");
427
- Constructor<?> constructor = timestampClass.getConstructor(Timestamp.class);
428
- return constructor.newInstance(toTimestamp(s, timeZone));
429
- }
430
-
431
-
432
- private List<List<Object>> select(String table) throws SQLException
433
- {
434
- try (Connection connection = connect()) {
435
- try (Statement statement = connection.createStatement()) {
436
- List<List<Object>> rows = new ArrayList<List<Object>>();
437
- String sql = "SELECT * FROM " + table;
438
- System.out.println(sql);
439
- try (ResultSet resultSet = statement.executeQuery(sql)) {
440
- while (resultSet.next()) {
441
- List<Object> row = new ArrayList<Object>();
442
- for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
443
- Object value = resultSet.getObject(i);
444
- if (value != null && value.getClass().getName().equals("oracle.sql.CLOB")) {
445
- value = resultSet.getString(i);
446
- }
447
- row.add(value);
448
- }
449
- rows.add(row);
450
- }
451
- }
452
- // cannot sort by CLOB, so sort by Java
453
- Collections.sort(rows, new Comparator<List<Object>>() {
454
- @Override
455
- public int compare(List<Object> o1, List<Object> o2) {
456
- return o1.toString().compareTo(o2.toString());
457
- }
458
- });
459
- return rows;
460
- }
461
- }
462
-
463
- }
464
-
465
-
466
- private void executeSQL(String sql) throws SQLException
467
- {
468
- executeSQL(sql, false);
469
- }
470
-
471
- private void executeSQL(String sql, boolean ignoreError) throws SQLException
472
- {
473
- try (Connection connection = connect()) {
474
- try {
475
- connection.setAutoCommit(true);
476
-
477
- try (Statement statement = connection.createStatement()) {
478
- System.out.println(String.format("Execute SQL : \"%s\".", sql));
479
- statement.execute(sql);
480
- }
481
-
482
- } catch (SQLException e) {
483
- if (!ignoreError) {
484
- throw e;
485
- }
486
- }
487
- }
488
- }
489
-
490
- private static Connection connect() throws SQLException
491
- {
492
- return DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TESTDB", "TEST_USER", "test_pw");
493
- }
494
-
495
- private void run(String ymlName) throws Exception
496
- {
497
- tester.run(convertYml(ymlName));
498
- }
499
-
500
- private String convertYml(String ymlName)
501
- {
502
- try {
503
- File ymlPath = convertPath(ymlName);
504
- File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
505
- Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
506
- try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
507
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
508
- String line;
509
- while ((line = reader.readLine()) != null) {
510
- Matcher matcher = pathPrefixPattern.matcher(line);
511
- if (matcher.matches()) {
512
- int group = 2;
513
- writer.write(line.substring(0, matcher.start(group)));
514
- writer.write(convertPath(matcher.group(group)).getAbsolutePath());
515
- writer.write(line.substring(matcher.end(group)));
516
- } else {
517
- writer.write(line);
518
- }
519
- writer.newLine();
520
- }
521
- }
522
- }
523
- return tempYmlPath.getAbsolutePath();
524
-
525
- } catch (IOException e) {
526
- throw new RuntimeException(e);
527
- }
528
- }
529
-
530
- private File convertPath(String name)
531
- {
532
- try {
533
- File root = new File(getClass().getResource("/dummy.txt").toURI()).getParentFile();
534
- return new File(root, name);
535
- } catch (URISyntaxException e) {
536
- throw new RuntimeException(e);
537
- }
538
- }
539
-
540
- }
1
+ package org.embulk.output.oracle;
2
+
3
+ import static org.junit.Assert.assertEquals;
4
+
5
+ import java.io.BufferedReader;
6
+ import java.io.BufferedWriter;
7
+ import java.io.File;
8
+ import java.io.FileReader;
9
+ import java.io.FileWriter;
10
+ import java.io.IOException;
11
+ import java.lang.reflect.Constructor;
12
+ import java.math.BigDecimal;
13
+ import java.net.URISyntaxException;
14
+ import java.sql.Connection;
15
+ import java.sql.DriverManager;
16
+ import java.sql.ResultSet;
17
+ import java.sql.SQLException;
18
+ import java.sql.Statement;
19
+ import java.sql.Timestamp;
20
+ import java.text.DateFormat;
21
+ import java.text.ParseException;
22
+ import java.text.SimpleDateFormat;
23
+ import java.util.ArrayList;
24
+ import java.util.Collections;
25
+ import java.util.Comparator;
26
+ import java.util.Date;
27
+ import java.util.Iterator;
28
+ import java.util.List;
29
+ import java.util.TimeZone;
30
+ import java.util.regex.Matcher;
31
+ import java.util.regex.Pattern;
32
+
33
+ import org.embulk.input.filesplit.LocalFileSplitInputPlugin;
34
+ import org.embulk.output.tester.EmbulkPluginTester;
35
+ import org.embulk.spi.InputPlugin;
36
+
37
+
38
+ public class OracleOutputPluginTestImpl
39
+ {
40
+ private EmbulkPluginTester tester;
41
+ private String pluginName;
42
+
43
+ public void setTester(EmbulkPluginTester tester) {
44
+ this.tester = tester;
45
+ }
46
+
47
+ public void setPluginName(String pluginName) {
48
+ this.pluginName = pluginName;
49
+ }
50
+
51
+ public String beforeClass()
52
+ {
53
+ try {
54
+ Class.forName("oracle.jdbc.OracleDriver");
55
+
56
+ try (Connection connection = connect()) {
57
+ String version = connection.getMetaData().getDriverVersion();
58
+ System.out.println("Driver version = " + version);
59
+ return version;
60
+ }
61
+
62
+ } catch (ClassNotFoundException | NoClassDefFoundError e) {
63
+ //throw new RuntimeException("You should put Oracle JDBC driver on 'driver' directory.");
64
+ System.err.println("Warning: put Oracle JDBC driver on 'driver' directory in order to test embulk-output-oracle plugin.");
65
+
66
+ } catch (SQLException e) {
67
+ System.err.println(e);
68
+ //throw new RuntimeException("You should prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
69
+ System.err.println("Warning: prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
70
+ // for example
71
+ // CREATE USER EMBULK_USER IDENTIFIED BY "embulk_pw";
72
+ // GRANT DBA TO EMBULK_USER;
73
+ }
74
+
75
+ convertPath("/data/test2/").mkdirs();
76
+
77
+ return null;
78
+ }
79
+
80
+ public void testInsert() throws Exception
81
+ {
82
+ String table = "TEST1";
83
+
84
+ dropTable(table);
85
+ createTable(table);
86
+
87
+ run("/yml/test-insert.yml");
88
+
89
+ assertTable(table);
90
+ }
91
+
92
+ public void testInsertEmpty() throws Exception
93
+ {
94
+ String table = "TEST1";
95
+
96
+ dropTable(table);
97
+ createTable(table);
98
+
99
+ run("/yml/test-insert-empty.yml");
100
+
101
+ assertTableEmpty(table);
102
+ }
103
+
104
+ public void testInsertDirect() throws Exception
105
+ {
106
+ String table = "TEST1";
107
+
108
+ dropTable(table);
109
+ createTable(table);
110
+
111
+ run("/yml/test-insert-direct.yml");
112
+
113
+ assertTable(table);
114
+ }
115
+
116
+ public void testInsertDirectEmpty() throws Exception
117
+ {
118
+ String table = "TEST1";
119
+
120
+ dropTable(table);
121
+ createTable(table);
122
+
123
+ run("/yml/test-insert-direct-empty.yml");
124
+
125
+ assertTableEmpty(table);
126
+ }
127
+
128
+ public void testInsertDirectCreate() throws Exception
129
+ {
130
+ String table = "TEST1";
131
+
132
+ dropTable(table);
133
+
134
+ run("/yml/test-insert-direct.yml");
135
+
136
+ assertGeneratedTable1(table);
137
+ }
138
+
139
+ public void testInsertDirectDirectMethod() throws Exception
140
+ {
141
+ String table = "TEST1";
142
+
143
+ dropTable(table);
144
+ createTable(table);
145
+
146
+ run("/yml/test-insert-direct-direct-method.yml");
147
+
148
+ assertTable(table);
149
+ }
150
+
151
+ public void testInsertDirectOCIMethod() throws Exception
152
+ {
153
+ String table = "TEST1";
154
+
155
+ dropTable(table);
156
+ createTable(table);
157
+
158
+ run("/yml/test-insert-direct-oci-method.yml");
159
+
160
+ assertTable(table);
161
+ }
162
+
163
+ public void testInsertDirectOCIMethodSplit() throws Exception
164
+ {
165
+ tester.addPlugin(InputPlugin.class, "filesplit", LocalFileSplitInputPlugin.class);
166
+
167
+ String table = "TEST1";
168
+
169
+ dropTable(table);
170
+ createTable(table);
171
+
172
+ run("/yml/test-insert-direct-oci-method-split.yml");
173
+
174
+ assertTable(table);
175
+ }
176
+
177
+ public void testUrl() throws Exception
178
+ {
179
+ String table = "TEST1";
180
+
181
+ dropTable(table);
182
+ createTable(table);
183
+
184
+ run("/yml/test-url.yml");
185
+
186
+ assertTable(table);
187
+ }
188
+
189
+ public void testReplace() throws Exception
190
+ {
191
+ String table = "TEST1";
192
+
193
+ dropTable(table);
194
+ createTable(table);
195
+
196
+ run("/yml/test-replace.yml");
197
+
198
+ assertGeneratedTable2(table);
199
+ }
200
+
201
+ public void testReplaceEmpty() throws Exception
202
+ {
203
+ String table = "TEST1";
204
+
205
+ dropTable(table);
206
+ createTable(table);
207
+
208
+ run("/yml/test-replace-empty.yml");
209
+
210
+ assertTableEmpty(table);
211
+ }
212
+
213
+ public void testReplaceCreate() throws Exception
214
+ {
215
+ String table = "TEST1";
216
+
217
+ dropTable(table);
218
+
219
+ run("/yml/test-replace.yml");
220
+
221
+ assertGeneratedTable2(table);
222
+ }
223
+
224
+
225
+ public void testReplaceLongName() throws Exception
226
+ {
227
+ String table = "TEST12345678901234567890123456";
228
+
229
+ dropTable(table);
230
+ createTable(table);
231
+
232
+ run("/yml/test-replace-long-name.yml");
233
+
234
+ assertGeneratedTable2(table);
235
+ }
236
+
237
+ public void testReplaceLongNameMultibyte() throws Exception
238
+ {
239
+ String table = "TEST12345678901234567890";
240
+
241
+ run("/yml/test-replace-long-name-multibyte.yml");
242
+
243
+ assertGeneratedTable2(table);
244
+ }
245
+
246
+ public void testStringTimestamp() throws Exception
247
+ {
248
+ String table = "TEST1";
249
+
250
+ dropTable(table);
251
+ createTable(table);
252
+
253
+ run("/yml/test-string-timestamp.yml");
254
+
255
+ assertTable(table);
256
+ }
257
+
258
+ private void dropTable(String table) throws SQLException
259
+ {
260
+ String sql = String.format("DROP TABLE %s", table);
261
+ executeSQL(sql, true);
262
+ }
263
+
264
+ private void createTable(String table) throws SQLException
265
+ {
266
+ String sql = String.format("CREATE TABLE %s ("
267
+ + "ID CHAR(4),"
268
+ + "VARCHAR2_ITEM VARCHAR2(40),"
269
+ + "INTEGER_ITEM NUMBER(4,0),"
270
+ + "NUMBER_ITEM NUMBER(10,2),"
271
+ + "DATE_ITEM DATE,"
272
+ + "TIMESTAMP_ITEM TIMESTAMP,"
273
+ + "PRIMARY KEY (ID))", table);
274
+ executeSQL(sql);
275
+ }
276
+
277
+ private void assertTable(String table) throws Exception
278
+ {
279
+ // datetime of UTC will be inserted by embulk.
280
+ // datetime of default timezone will be selected by JDBC.
281
+ TimeZone timeZone = TimeZone.getDefault();
282
+ List<List<Object>> rows = select(table);
283
+
284
+ /*
285
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
286
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
287
+ A003,,,,,
288
+ */
289
+
290
+ assertEquals(3, rows.size());
291
+ Iterator<List<Object>> i1 = rows.iterator();
292
+ {
293
+ Iterator<Object> i2 = i1.next().iterator();
294
+ assertEquals("A001", i2.next());
295
+ assertEquals("ABCDE", i2.next());
296
+ assertEquals(new BigDecimal("0"), i2.next());
297
+ assertEquals(new BigDecimal("123.45"), i2.next());
298
+ assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
299
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
300
+ }
301
+ {
302
+ Iterator<Object> i2 = i1.next().iterator();
303
+ assertEquals("A002", i2.next());
304
+ assertEquals("あいうえお", i2.next());
305
+ assertEquals(new BigDecimal("-9999"), i2.next());
306
+ assertEquals(new BigDecimal("-99999999.99"), i2.next());
307
+ assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
308
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
309
+ }
310
+ {
311
+ Iterator<Object> i2 = i1.next().iterator();
312
+ assertEquals("A003", i2.next());
313
+ assertEquals(null, i2.next());
314
+ assertEquals(null, i2.next());
315
+ assertEquals(null, i2.next());
316
+ assertEquals(null, i2.next());
317
+ assertEquals(null, i2.next());
318
+ }
319
+ }
320
+
321
+ private void assertTableEmpty(String table) throws Exception
322
+ {
323
+ List<List<Object>> rows = select(table);
324
+ assertEquals(0, rows.size());
325
+ }
326
+
327
+ private void assertGeneratedTable1(String table) throws Exception
328
+ {
329
+ // datetime of UTC will be inserted by embulk.
330
+ // datetime of default timezone will be selected by JDBC.
331
+ TimeZone timeZone = TimeZone.getDefault();
332
+ List<List<Object>> rows = select(table);
333
+
334
+ /*
335
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
336
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
337
+ A003,,,,,
338
+ */
339
+
340
+ assertEquals(3, rows.size());
341
+ Iterator<List<Object>> i1 = rows.iterator();
342
+ {
343
+ Iterator<Object> i2 = i1.next().iterator();
344
+ assertEquals("A001", i2.next());
345
+ assertEquals("ABCDE", i2.next());
346
+ assertEquals(new BigDecimal("0"), i2.next());
347
+ assertEquals("123.45", i2.next());
348
+ assertEquals(toOracleTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
349
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
350
+ }
351
+ {
352
+ Iterator<Object> i2 = i1.next().iterator();
353
+ assertEquals("A002", i2.next());
354
+ assertEquals("あいうえお", i2.next());
355
+ assertEquals(new BigDecimal("-9999"), i2.next());
356
+ assertEquals("-99999999.99", i2.next());
357
+ assertEquals(toOracleTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
358
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
359
+ }
360
+ {
361
+ Iterator<Object> i2 = i1.next().iterator();
362
+ assertEquals("A003", i2.next());
363
+ assertEquals(null, i2.next());
364
+ assertEquals(null, i2.next());
365
+ assertEquals(null, i2.next());
366
+ assertEquals(null, i2.next());
367
+ assertEquals(null, i2.next());
368
+ }
369
+ }
370
+
371
+ private void assertGeneratedTable2(String table) throws Exception
372
+ {
373
+ // datetime of UTC will be inserted by embulk.
374
+ // datetime of default timezone will be selected by JDBC.
375
+ TimeZone timeZone = TimeZone.getDefault();
376
+ List<List<Object>> rows = select(table);
377
+
378
+ /*
379
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
380
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
381
+ A003,,,,,
382
+ */
383
+
384
+ assertEquals(3, rows.size());
385
+ Iterator<List<Object>> i1 = rows.iterator();
386
+ {
387
+ Iterator<Object> i2 = i1.next().iterator();
388
+ assertEquals("A001", i2.next());
389
+ assertEquals("ABCDE", i2.next());
390
+ assertEquals(new BigDecimal("0"), i2.next());
391
+ assertEquals(new BigDecimal("123.45"), i2.next());
392
+ assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
393
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
394
+ }
395
+ {
396
+ Iterator<Object> i2 = i1.next().iterator();
397
+ assertEquals("A002", i2.next());
398
+ assertEquals("あいうえお", i2.next());
399
+ assertEquals(new BigDecimal("-9999"), i2.next());
400
+ assertEquals(new BigDecimal("-99999999.99"), i2.next());
401
+ assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
402
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
403
+ }
404
+ {
405
+ Iterator<Object> i2 = i1.next().iterator();
406
+ assertEquals("A003", i2.next());
407
+ assertEquals(null, i2.next());
408
+ assertEquals(null, i2.next());
409
+ assertEquals(null, i2.next());
410
+ assertEquals(null, i2.next());
411
+ assertEquals(null, i2.next());
412
+ }
413
+ }
414
+
415
+
416
+ private Timestamp toTimestamp(String s, TimeZone timeZone)
417
+ {
418
+ for (String formatString : new String[]{"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd"}) {
419
+ DateFormat dateFormat = new SimpleDateFormat(formatString);
420
+ dateFormat.setTimeZone(timeZone);
421
+ try {
422
+ Date date = dateFormat.parse(s);
423
+ return new Timestamp(date.getTime());
424
+ } catch (ParseException e) {
425
+ // NOP
426
+ }
427
+ }
428
+ throw new IllegalArgumentException(s);
429
+ }
430
+
431
+ private Object toOracleTimestamp(String s, TimeZone timeZone) throws Exception
432
+ {
433
+ Class<?> timestampClass = Class.forName("oracle.sql.TIMESTAMP");
434
+ Constructor<?> constructor = timestampClass.getConstructor(Timestamp.class);
435
+ return constructor.newInstance(toTimestamp(s, timeZone));
436
+ }
437
+
438
+
439
+ private List<List<Object>> select(String table) throws SQLException
440
+ {
441
+ try (Connection connection = connect()) {
442
+ try (Statement statement = connection.createStatement()) {
443
+ List<List<Object>> rows = new ArrayList<List<Object>>();
444
+ String sql = "SELECT * FROM " + table;
445
+ System.out.println(sql);
446
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
447
+ while (resultSet.next()) {
448
+ List<Object> row = new ArrayList<Object>();
449
+ for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
450
+ Object value = resultSet.getObject(i);
451
+ if (value != null && value.getClass().getName().equals("oracle.sql.CLOB")) {
452
+ value = resultSet.getString(i);
453
+ }
454
+ row.add(value);
455
+ }
456
+ rows.add(row);
457
+ }
458
+ }
459
+ // cannot sort by CLOB, so sort by Java
460
+ Collections.sort(rows, new Comparator<List<Object>>() {
461
+ @Override
462
+ public int compare(List<Object> o1, List<Object> o2) {
463
+ return o1.toString().compareTo(o2.toString());
464
+ }
465
+ });
466
+ return rows;
467
+ }
468
+ }
469
+
470
+ }
471
+
472
+
473
+ private void executeSQL(String sql) throws SQLException
474
+ {
475
+ executeSQL(sql, false);
476
+ }
477
+
478
+ private void executeSQL(String sql, boolean ignoreError) throws SQLException
479
+ {
480
+ try (Connection connection = connect()) {
481
+ try {
482
+ connection.setAutoCommit(true);
483
+
484
+ try (Statement statement = connection.createStatement()) {
485
+ System.out.println(String.format("Execute SQL : \"%s\".", sql));
486
+ statement.execute(sql);
487
+ }
488
+
489
+ } catch (SQLException e) {
490
+ if (!ignoreError) {
491
+ throw e;
492
+ }
493
+ }
494
+ }
495
+ }
496
+
497
+ private static Connection connect() throws SQLException
498
+ {
499
+ return DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TESTDB", "TEST_USER", "test_pw");
500
+ }
501
+
502
+ private void run(String ymlName) throws Exception
503
+ {
504
+ tester.run(convertYml(ymlName));
505
+ }
506
+
507
+ private String convertYml(String ymlName)
508
+ {
509
+ try {
510
+ File ymlPath = convertPath(ymlName);
511
+ File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
512
+ Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
513
+ try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
514
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
515
+ String line;
516
+ while ((line = reader.readLine()) != null) {
517
+ line = line.replaceAll("type: oracle", "type: " + pluginName);
518
+ Matcher matcher = pathPrefixPattern.matcher(line);
519
+ if (matcher.matches()) {
520
+ int group = 2;
521
+ writer.write(line.substring(0, matcher.start(group)));
522
+ writer.write(convertPath(matcher.group(group)).getAbsolutePath());
523
+ writer.write(line.substring(matcher.end(group)));
524
+ } else {
525
+ writer.write(line);
526
+ }
527
+ writer.newLine();
528
+ }
529
+ }
530
+ }
531
+ return tempYmlPath.getAbsolutePath();
532
+
533
+ } catch (IOException e) {
534
+ throw new RuntimeException(e);
535
+ }
536
+ }
537
+
538
+ private File convertPath(String name)
539
+ {
540
+ try {
541
+ File root = new File(getClass().getResource("/dummy.txt").toURI()).getParentFile();
542
+ return new File(root, name);
543
+ } catch (URISyntaxException e) {
544
+ throw new RuntimeException(e);
545
+ }
546
+ }
547
+
548
+ }