embulk-output-oracle 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +160 -160
  3. data/build.gradle +6 -6
  4. data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
  5. data/classpath/{embulk-output-oracle-0.4.1.jar → embulk-output-oracle-0.4.2.jar} +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -153
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +179 -165
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +185 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +50 -25
  37. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +548 -540
  38. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  39. data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java +79 -0
  40. data/src/test/resources/data/test1/test1.csv +3 -3
  41. data/src/test/resources/dummy.txt +1 -0
  42. data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -26
  43. data/src/test/resources/yml/test-insert-direct-empty.yml +25 -25
  44. data/src/test/resources/yml/test-insert-direct-oci-method-split.yml +28 -28
  45. data/src/test/resources/yml/test-insert-direct-oci-method.yml +28 -28
  46. data/src/test/resources/yml/test-insert-direct.yml +25 -25
  47. data/src/test/resources/yml/test-insert-empty.yml +27 -27
  48. data/src/test/resources/yml/test-insert.yml +25 -27
  49. data/src/test/resources/yml/test-replace-empty.yml +31 -31
  50. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
  51. data/src/test/resources/yml/test-replace-long-name.yml +31 -31
  52. data/src/test/resources/yml/test-replace.yml +31 -31
  53. data/src/test/resources/yml/test-string-timestamp.yml +28 -28
  54. data/src/test/resources/yml/test-url.yml +24 -24
  55. metadata +6 -8
  56. data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
  57. data/lib/embulk/linux_x64/libembulk-output-oracle.so +0 -0
  58. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +0 -42
  59. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +0 -124
  60. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +0 -105
@@ -1,540 +1,548 @@
1
- package org.embulk.output.oracle;
2
-
3
- import static org.junit.Assert.assertEquals;
4
-
5
- import java.io.BufferedReader;
6
- import java.io.BufferedWriter;
7
- import java.io.File;
8
- import java.io.FileReader;
9
- import java.io.FileWriter;
10
- import java.io.IOException;
11
- import java.lang.reflect.Constructor;
12
- import java.math.BigDecimal;
13
- import java.net.URISyntaxException;
14
- import java.sql.Connection;
15
- import java.sql.DriverManager;
16
- import java.sql.ResultSet;
17
- import java.sql.SQLException;
18
- import java.sql.Statement;
19
- import java.sql.Timestamp;
20
- import java.text.DateFormat;
21
- import java.text.ParseException;
22
- import java.text.SimpleDateFormat;
23
- import java.util.ArrayList;
24
- import java.util.Collections;
25
- import java.util.Comparator;
26
- import java.util.Date;
27
- import java.util.Iterator;
28
- import java.util.List;
29
- import java.util.TimeZone;
30
- import java.util.regex.Matcher;
31
- import java.util.regex.Pattern;
32
-
33
- import org.embulk.input.filesplit.LocalFileSplitInputPlugin;
34
- import org.embulk.output.OracleOutputPlugin;
35
- import org.embulk.spi.InputPlugin;
36
- import org.embulk.spi.OutputPlugin;
37
-
38
-
39
- public class OracleOutputPluginTestImpl
40
- {
41
- private EmbulkPluginTester tester = new EmbulkPluginTester(OutputPlugin.class, "oracle", OracleOutputPlugin.class);
42
-
43
-
44
- public String beforeClass()
45
- {
46
- try {
47
- Class.forName("oracle.jdbc.OracleDriver");
48
-
49
- try (Connection connection = connect()) {
50
- String version = connection.getMetaData().getDriverVersion();
51
- System.out.println("Driver version = " + version);
52
- return version;
53
- }
54
-
55
- } catch (ClassNotFoundException | NoClassDefFoundError e) {
56
- //throw new RuntimeException("You should put Oracle JDBC driver on 'driver' directory.");
57
- System.err.println("Warning: put Oracle JDBC driver on 'driver' directory in order to test embulk-output-oracle plugin.");
58
-
59
- } catch (SQLException e) {
60
- System.err.println(e);
61
- //throw new RuntimeException("You should prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
62
- System.err.println("Warning: prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
63
- // for example
64
- // CREATE USER EMBULK_USER IDENTIFIED BY "embulk_pw";
65
- // GRANT DBA TO EMBULK_USER;
66
- }
67
-
68
- convertPath("/data/test2/").mkdirs();
69
-
70
- return null;
71
- }
72
-
73
- public void testInsert() throws Exception
74
- {
75
- String table = "TEST1";
76
-
77
- dropTable(table);
78
- createTable(table);
79
-
80
- run("/yml/test-insert.yml");
81
-
82
- assertTable(table);
83
- }
84
-
85
- public void testInsertEmpty() throws Exception
86
- {
87
- String table = "TEST1";
88
-
89
- dropTable(table);
90
- createTable(table);
91
-
92
- run("/yml/test-insert-empty.yml");
93
-
94
- assertTableEmpty(table);
95
- }
96
-
97
- public void testInsertDirect() throws Exception
98
- {
99
- String table = "TEST1";
100
-
101
- dropTable(table);
102
- createTable(table);
103
-
104
- run("/yml/test-insert-direct.yml");
105
-
106
- assertTable(table);
107
- }
108
-
109
- public void testInsertDirectEmpty() throws Exception
110
- {
111
- String table = "TEST1";
112
-
113
- dropTable(table);
114
- createTable(table);
115
-
116
- run("/yml/test-insert-direct-empty.yml");
117
-
118
- assertTableEmpty(table);
119
- }
120
-
121
- public void testInsertDirectCreate() throws Exception
122
- {
123
- String table = "TEST1";
124
-
125
- dropTable(table);
126
-
127
- run("/yml/test-insert-direct.yml");
128
-
129
- assertGeneratedTable1(table);
130
- }
131
-
132
- public void testInsertDirectDirectMethod() throws Exception
133
- {
134
- String table = "TEST1";
135
-
136
- dropTable(table);
137
- createTable(table);
138
-
139
- run("/yml/test-insert-direct-direct-method.yml");
140
-
141
- assertTable(table);
142
- }
143
-
144
- public void testInsertDirectOCIMethod() throws Exception
145
- {
146
- String table = "TEST1";
147
-
148
- dropTable(table);
149
- createTable(table);
150
-
151
- run("/yml/test-insert-direct-oci-method.yml");
152
-
153
- assertTable(table);
154
- }
155
-
156
- public void testInsertDirectOCIMethodSplit() throws Exception
157
- {
158
- tester.addPlugin(InputPlugin.class, "filesplit", LocalFileSplitInputPlugin.class);
159
-
160
- String table = "TEST1";
161
-
162
- dropTable(table);
163
- createTable(table);
164
-
165
- run("/yml/test-insert-direct-oci-method-split.yml");
166
-
167
- assertTable(table);
168
- }
169
-
170
- public void testUrl() throws Exception
171
- {
172
- String table = "TEST1";
173
-
174
- dropTable(table);
175
- createTable(table);
176
-
177
- run("/yml/test-url.yml");
178
-
179
- assertTable(table);
180
- }
181
-
182
- public void testReplace() throws Exception
183
- {
184
- String table = "TEST1";
185
-
186
- dropTable(table);
187
- createTable(table);
188
-
189
- run("/yml/test-replace.yml");
190
-
191
- assertGeneratedTable2(table);
192
- }
193
-
194
- public void testReplaceEmpty() throws Exception
195
- {
196
- String table = "TEST1";
197
-
198
- dropTable(table);
199
- createTable(table);
200
-
201
- run("/yml/test-replace-empty.yml");
202
-
203
- assertTableEmpty(table);
204
- }
205
-
206
- public void testReplaceCreate() throws Exception
207
- {
208
- String table = "TEST1";
209
-
210
- dropTable(table);
211
-
212
- run("/yml/test-replace.yml");
213
-
214
- assertGeneratedTable2(table);
215
- }
216
-
217
-
218
- public void testReplaceLongName() throws Exception
219
- {
220
- String table = "TEST12345678901234567890123456";
221
-
222
- dropTable(table);
223
- createTable(table);
224
-
225
- run("/yml/test-replace-long-name.yml");
226
-
227
- assertGeneratedTable2(table);
228
- }
229
-
230
- public void testReplaceLongNameMultibyte() throws Exception
231
- {
232
- String table = "TEST12345678901234567890";
233
-
234
- run("/yml/test-replace-long-name-multibyte.yml");
235
-
236
- assertGeneratedTable2(table);
237
- }
238
-
239
- public void testStringTimestamp() throws Exception
240
- {
241
- String table = "TEST1";
242
-
243
- dropTable(table);
244
- createTable(table);
245
-
246
- run("/yml/test-string-timestamp.yml");
247
-
248
- assertTable(table);
249
- }
250
-
251
- private void dropTable(String table) throws SQLException
252
- {
253
- String sql = String.format("DROP TABLE %s", table);
254
- executeSQL(sql, true);
255
- }
256
-
257
- private void createTable(String table) throws SQLException
258
- {
259
- String sql = String.format("CREATE TABLE %s ("
260
- + "ID CHAR(4),"
261
- + "VARCHAR2_ITEM VARCHAR2(40),"
262
- + "INTEGER_ITEM NUMBER(4,0),"
263
- + "NUMBER_ITEM NUMBER(10,2),"
264
- + "DATE_ITEM DATE,"
265
- + "TIMESTAMP_ITEM TIMESTAMP,"
266
- + "PRIMARY KEY (ID))", table);
267
- executeSQL(sql);
268
- }
269
-
270
- private void assertTable(String table) throws Exception
271
- {
272
- // datetime of UTC will be inserted by embulk.
273
- // datetime of default timezone will be selected by JDBC.
274
- TimeZone timeZone = TimeZone.getDefault();
275
- List<List<Object>> rows = select(table);
276
-
277
- /*
278
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
279
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
280
- A003,,,,,
281
- */
282
-
283
- assertEquals(3, rows.size());
284
- Iterator<List<Object>> i1 = rows.iterator();
285
- {
286
- Iterator<Object> i2 = i1.next().iterator();
287
- assertEquals("A001", i2.next());
288
- assertEquals("ABCDE", i2.next());
289
- assertEquals(new BigDecimal("0"), i2.next());
290
- assertEquals(new BigDecimal("123.45"), i2.next());
291
- assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
292
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
293
- }
294
- {
295
- Iterator<Object> i2 = i1.next().iterator();
296
- assertEquals("A002", i2.next());
297
- assertEquals("あいうえお", i2.next());
298
- assertEquals(new BigDecimal("-9999"), i2.next());
299
- assertEquals(new BigDecimal("-99999999.99"), i2.next());
300
- assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
301
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
302
- }
303
- {
304
- Iterator<Object> i2 = i1.next().iterator();
305
- assertEquals("A003", i2.next());
306
- assertEquals(null, i2.next());
307
- assertEquals(null, i2.next());
308
- assertEquals(null, i2.next());
309
- assertEquals(null, i2.next());
310
- assertEquals(null, i2.next());
311
- }
312
- }
313
-
314
- private void assertTableEmpty(String table) throws Exception
315
- {
316
- List<List<Object>> rows = select(table);
317
- assertEquals(0, rows.size());
318
- }
319
-
320
- private void assertGeneratedTable1(String table) throws Exception
321
- {
322
- // datetime of UTC will be inserted by embulk.
323
- // datetime of default timezone will be selected by JDBC.
324
- TimeZone timeZone = TimeZone.getDefault();
325
- List<List<Object>> rows = select(table);
326
-
327
- /*
328
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
329
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
330
- A003,,,,,
331
- */
332
-
333
- assertEquals(3, rows.size());
334
- Iterator<List<Object>> i1 = rows.iterator();
335
- {
336
- Iterator<Object> i2 = i1.next().iterator();
337
- assertEquals("A001", i2.next());
338
- assertEquals("ABCDE", i2.next());
339
- assertEquals(new BigDecimal("0"), i2.next());
340
- assertEquals("123.45", i2.next());
341
- assertEquals(toOracleTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
342
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
343
- }
344
- {
345
- Iterator<Object> i2 = i1.next().iterator();
346
- assertEquals("A002", i2.next());
347
- assertEquals("あいうえお", i2.next());
348
- assertEquals(new BigDecimal("-9999"), i2.next());
349
- assertEquals("-99999999.99", i2.next());
350
- assertEquals(toOracleTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
351
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
352
- }
353
- {
354
- Iterator<Object> i2 = i1.next().iterator();
355
- assertEquals("A003", i2.next());
356
- assertEquals(null, i2.next());
357
- assertEquals(null, i2.next());
358
- assertEquals(null, i2.next());
359
- assertEquals(null, i2.next());
360
- assertEquals(null, i2.next());
361
- }
362
- }
363
-
364
- private void assertGeneratedTable2(String table) throws Exception
365
- {
366
- // datetime of UTC will be inserted by embulk.
367
- // datetime of default timezone will be selected by JDBC.
368
- TimeZone timeZone = TimeZone.getDefault();
369
- List<List<Object>> rows = select(table);
370
-
371
- /*
372
- A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
373
- A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
374
- A003,,,,,
375
- */
376
-
377
- assertEquals(3, rows.size());
378
- Iterator<List<Object>> i1 = rows.iterator();
379
- {
380
- Iterator<Object> i2 = i1.next().iterator();
381
- assertEquals("A001", i2.next());
382
- assertEquals("ABCDE", i2.next());
383
- assertEquals(new BigDecimal("0"), i2.next());
384
- assertEquals(new BigDecimal("123.45"), i2.next());
385
- assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
386
- assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
387
- }
388
- {
389
- Iterator<Object> i2 = i1.next().iterator();
390
- assertEquals("A002", i2.next());
391
- assertEquals("あいうえお", i2.next());
392
- assertEquals(new BigDecimal("-9999"), i2.next());
393
- assertEquals(new BigDecimal("-99999999.99"), i2.next());
394
- assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
395
- assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
396
- }
397
- {
398
- Iterator<Object> i2 = i1.next().iterator();
399
- assertEquals("A003", i2.next());
400
- assertEquals(null, i2.next());
401
- assertEquals(null, i2.next());
402
- assertEquals(null, i2.next());
403
- assertEquals(null, i2.next());
404
- assertEquals(null, i2.next());
405
- }
406
- }
407
-
408
-
409
- private Timestamp toTimestamp(String s, TimeZone timeZone)
410
- {
411
- for (String formatString : new String[]{"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd"}) {
412
- DateFormat dateFormat = new SimpleDateFormat(formatString);
413
- dateFormat.setTimeZone(timeZone);
414
- try {
415
- Date date = dateFormat.parse(s);
416
- return new Timestamp(date.getTime());
417
- } catch (ParseException e) {
418
- // NOP
419
- }
420
- }
421
- throw new IllegalArgumentException(s);
422
- }
423
-
424
- private Object toOracleTimestamp(String s, TimeZone timeZone) throws Exception
425
- {
426
- Class<?> timestampClass = Class.forName("oracle.sql.TIMESTAMP");
427
- Constructor<?> constructor = timestampClass.getConstructor(Timestamp.class);
428
- return constructor.newInstance(toTimestamp(s, timeZone));
429
- }
430
-
431
-
432
- private List<List<Object>> select(String table) throws SQLException
433
- {
434
- try (Connection connection = connect()) {
435
- try (Statement statement = connection.createStatement()) {
436
- List<List<Object>> rows = new ArrayList<List<Object>>();
437
- String sql = "SELECT * FROM " + table;
438
- System.out.println(sql);
439
- try (ResultSet resultSet = statement.executeQuery(sql)) {
440
- while (resultSet.next()) {
441
- List<Object> row = new ArrayList<Object>();
442
- for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
443
- Object value = resultSet.getObject(i);
444
- if (value != null && value.getClass().getName().equals("oracle.sql.CLOB")) {
445
- value = resultSet.getString(i);
446
- }
447
- row.add(value);
448
- }
449
- rows.add(row);
450
- }
451
- }
452
- // cannot sort by CLOB, so sort by Java
453
- Collections.sort(rows, new Comparator<List<Object>>() {
454
- @Override
455
- public int compare(List<Object> o1, List<Object> o2) {
456
- return o1.toString().compareTo(o2.toString());
457
- }
458
- });
459
- return rows;
460
- }
461
- }
462
-
463
- }
464
-
465
-
466
- private void executeSQL(String sql) throws SQLException
467
- {
468
- executeSQL(sql, false);
469
- }
470
-
471
- private void executeSQL(String sql, boolean ignoreError) throws SQLException
472
- {
473
- try (Connection connection = connect()) {
474
- try {
475
- connection.setAutoCommit(true);
476
-
477
- try (Statement statement = connection.createStatement()) {
478
- System.out.println(String.format("Execute SQL : \"%s\".", sql));
479
- statement.execute(sql);
480
- }
481
-
482
- } catch (SQLException e) {
483
- if (!ignoreError) {
484
- throw e;
485
- }
486
- }
487
- }
488
- }
489
-
490
- private static Connection connect() throws SQLException
491
- {
492
- return DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TESTDB", "TEST_USER", "test_pw");
493
- }
494
-
495
- private void run(String ymlName) throws Exception
496
- {
497
- tester.run(convertYml(ymlName));
498
- }
499
-
500
- private String convertYml(String ymlName)
501
- {
502
- try {
503
- File ymlPath = convertPath(ymlName);
504
- File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
505
- Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
506
- try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
507
- try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
508
- String line;
509
- while ((line = reader.readLine()) != null) {
510
- Matcher matcher = pathPrefixPattern.matcher(line);
511
- if (matcher.matches()) {
512
- int group = 2;
513
- writer.write(line.substring(0, matcher.start(group)));
514
- writer.write(convertPath(matcher.group(group)).getAbsolutePath());
515
- writer.write(line.substring(matcher.end(group)));
516
- } else {
517
- writer.write(line);
518
- }
519
- writer.newLine();
520
- }
521
- }
522
- }
523
- return tempYmlPath.getAbsolutePath();
524
-
525
- } catch (IOException e) {
526
- throw new RuntimeException(e);
527
- }
528
- }
529
-
530
- private File convertPath(String name)
531
- {
532
- try {
533
- File root = new File(getClass().getResource("/dummy.txt").toURI()).getParentFile();
534
- return new File(root, name);
535
- } catch (URISyntaxException e) {
536
- throw new RuntimeException(e);
537
- }
538
- }
539
-
540
- }
1
+ package org.embulk.output.oracle;
2
+
3
+ import static org.junit.Assert.assertEquals;
4
+
5
+ import java.io.BufferedReader;
6
+ import java.io.BufferedWriter;
7
+ import java.io.File;
8
+ import java.io.FileReader;
9
+ import java.io.FileWriter;
10
+ import java.io.IOException;
11
+ import java.lang.reflect.Constructor;
12
+ import java.math.BigDecimal;
13
+ import java.net.URISyntaxException;
14
+ import java.sql.Connection;
15
+ import java.sql.DriverManager;
16
+ import java.sql.ResultSet;
17
+ import java.sql.SQLException;
18
+ import java.sql.Statement;
19
+ import java.sql.Timestamp;
20
+ import java.text.DateFormat;
21
+ import java.text.ParseException;
22
+ import java.text.SimpleDateFormat;
23
+ import java.util.ArrayList;
24
+ import java.util.Collections;
25
+ import java.util.Comparator;
26
+ import java.util.Date;
27
+ import java.util.Iterator;
28
+ import java.util.List;
29
+ import java.util.TimeZone;
30
+ import java.util.regex.Matcher;
31
+ import java.util.regex.Pattern;
32
+
33
+ import org.embulk.input.filesplit.LocalFileSplitInputPlugin;
34
+ import org.embulk.output.tester.EmbulkPluginTester;
35
+ import org.embulk.spi.InputPlugin;
36
+
37
+
38
+ public class OracleOutputPluginTestImpl
39
+ {
40
+ private EmbulkPluginTester tester;
41
+ private String pluginName;
42
+
43
+ public void setTester(EmbulkPluginTester tester) {
44
+ this.tester = tester;
45
+ }
46
+
47
+ public void setPluginName(String pluginName) {
48
+ this.pluginName = pluginName;
49
+ }
50
+
51
+ public String beforeClass()
52
+ {
53
+ try {
54
+ Class.forName("oracle.jdbc.OracleDriver");
55
+
56
+ try (Connection connection = connect()) {
57
+ String version = connection.getMetaData().getDriverVersion();
58
+ System.out.println("Driver version = " + version);
59
+ return version;
60
+ }
61
+
62
+ } catch (ClassNotFoundException | NoClassDefFoundError e) {
63
+ //throw new RuntimeException("You should put Oracle JDBC driver on 'driver' directory.");
64
+ System.err.println("Warning: put Oracle JDBC driver on 'driver' directory in order to test embulk-output-oracle plugin.");
65
+
66
+ } catch (SQLException e) {
67
+ System.err.println(e);
68
+ //throw new RuntimeException("You should prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
69
+ System.err.println("Warning: prepare a schema on Oracle (database = 'TESTDB', user = 'TEST_USER', password = 'test_pw').");
70
+ // for example
71
+ // CREATE USER EMBULK_USER IDENTIFIED BY "embulk_pw";
72
+ // GRANT DBA TO EMBULK_USER;
73
+ }
74
+
75
+ convertPath("/data/test2/").mkdirs();
76
+
77
+ return null;
78
+ }
79
+
80
+ public void testInsert() throws Exception
81
+ {
82
+ String table = "TEST1";
83
+
84
+ dropTable(table);
85
+ createTable(table);
86
+
87
+ run("/yml/test-insert.yml");
88
+
89
+ assertTable(table);
90
+ }
91
+
92
+ public void testInsertEmpty() throws Exception
93
+ {
94
+ String table = "TEST1";
95
+
96
+ dropTable(table);
97
+ createTable(table);
98
+
99
+ run("/yml/test-insert-empty.yml");
100
+
101
+ assertTableEmpty(table);
102
+ }
103
+
104
+ public void testInsertDirect() throws Exception
105
+ {
106
+ String table = "TEST1";
107
+
108
+ dropTable(table);
109
+ createTable(table);
110
+
111
+ run("/yml/test-insert-direct.yml");
112
+
113
+ assertTable(table);
114
+ }
115
+
116
+ public void testInsertDirectEmpty() throws Exception
117
+ {
118
+ String table = "TEST1";
119
+
120
+ dropTable(table);
121
+ createTable(table);
122
+
123
+ run("/yml/test-insert-direct-empty.yml");
124
+
125
+ assertTableEmpty(table);
126
+ }
127
+
128
+ public void testInsertDirectCreate() throws Exception
129
+ {
130
+ String table = "TEST1";
131
+
132
+ dropTable(table);
133
+
134
+ run("/yml/test-insert-direct.yml");
135
+
136
+ assertGeneratedTable1(table);
137
+ }
138
+
139
+ public void testInsertDirectDirectMethod() throws Exception
140
+ {
141
+ String table = "TEST1";
142
+
143
+ dropTable(table);
144
+ createTable(table);
145
+
146
+ run("/yml/test-insert-direct-direct-method.yml");
147
+
148
+ assertTable(table);
149
+ }
150
+
151
+ public void testInsertDirectOCIMethod() throws Exception
152
+ {
153
+ String table = "TEST1";
154
+
155
+ dropTable(table);
156
+ createTable(table);
157
+
158
+ run("/yml/test-insert-direct-oci-method.yml");
159
+
160
+ assertTable(table);
161
+ }
162
+
163
+ public void testInsertDirectOCIMethodSplit() throws Exception
164
+ {
165
+ tester.addPlugin(InputPlugin.class, "filesplit", LocalFileSplitInputPlugin.class);
166
+
167
+ String table = "TEST1";
168
+
169
+ dropTable(table);
170
+ createTable(table);
171
+
172
+ run("/yml/test-insert-direct-oci-method-split.yml");
173
+
174
+ assertTable(table);
175
+ }
176
+
177
+ public void testUrl() throws Exception
178
+ {
179
+ String table = "TEST1";
180
+
181
+ dropTable(table);
182
+ createTable(table);
183
+
184
+ run("/yml/test-url.yml");
185
+
186
+ assertTable(table);
187
+ }
188
+
189
+ public void testReplace() throws Exception
190
+ {
191
+ String table = "TEST1";
192
+
193
+ dropTable(table);
194
+ createTable(table);
195
+
196
+ run("/yml/test-replace.yml");
197
+
198
+ assertGeneratedTable2(table);
199
+ }
200
+
201
+ public void testReplaceEmpty() throws Exception
202
+ {
203
+ String table = "TEST1";
204
+
205
+ dropTable(table);
206
+ createTable(table);
207
+
208
+ run("/yml/test-replace-empty.yml");
209
+
210
+ assertTableEmpty(table);
211
+ }
212
+
213
+ public void testReplaceCreate() throws Exception
214
+ {
215
+ String table = "TEST1";
216
+
217
+ dropTable(table);
218
+
219
+ run("/yml/test-replace.yml");
220
+
221
+ assertGeneratedTable2(table);
222
+ }
223
+
224
+
225
+ public void testReplaceLongName() throws Exception
226
+ {
227
+ String table = "TEST12345678901234567890123456";
228
+
229
+ dropTable(table);
230
+ createTable(table);
231
+
232
+ run("/yml/test-replace-long-name.yml");
233
+
234
+ assertGeneratedTable2(table);
235
+ }
236
+
237
+ public void testReplaceLongNameMultibyte() throws Exception
238
+ {
239
+ String table = "TEST12345678901234567890";
240
+
241
+ run("/yml/test-replace-long-name-multibyte.yml");
242
+
243
+ assertGeneratedTable2(table);
244
+ }
245
+
246
+ public void testStringTimestamp() throws Exception
247
+ {
248
+ String table = "TEST1";
249
+
250
+ dropTable(table);
251
+ createTable(table);
252
+
253
+ run("/yml/test-string-timestamp.yml");
254
+
255
+ assertTable(table);
256
+ }
257
+
258
+ private void dropTable(String table) throws SQLException
259
+ {
260
+ String sql = String.format("DROP TABLE %s", table);
261
+ executeSQL(sql, true);
262
+ }
263
+
264
+ private void createTable(String table) throws SQLException
265
+ {
266
+ String sql = String.format("CREATE TABLE %s ("
267
+ + "ID CHAR(4),"
268
+ + "VARCHAR2_ITEM VARCHAR2(40),"
269
+ + "INTEGER_ITEM NUMBER(4,0),"
270
+ + "NUMBER_ITEM NUMBER(10,2),"
271
+ + "DATE_ITEM DATE,"
272
+ + "TIMESTAMP_ITEM TIMESTAMP,"
273
+ + "PRIMARY KEY (ID))", table);
274
+ executeSQL(sql);
275
+ }
276
+
277
+ private void assertTable(String table) throws Exception
278
+ {
279
+ // datetime of UTC will be inserted by embulk.
280
+ // datetime of default timezone will be selected by JDBC.
281
+ TimeZone timeZone = TimeZone.getDefault();
282
+ List<List<Object>> rows = select(table);
283
+
284
+ /*
285
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
286
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
287
+ A003,,,,,
288
+ */
289
+
290
+ assertEquals(3, rows.size());
291
+ Iterator<List<Object>> i1 = rows.iterator();
292
+ {
293
+ Iterator<Object> i2 = i1.next().iterator();
294
+ assertEquals("A001", i2.next());
295
+ assertEquals("ABCDE", i2.next());
296
+ assertEquals(new BigDecimal("0"), i2.next());
297
+ assertEquals(new BigDecimal("123.45"), i2.next());
298
+ assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
299
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
300
+ }
301
+ {
302
+ Iterator<Object> i2 = i1.next().iterator();
303
+ assertEquals("A002", i2.next());
304
+ assertEquals("あいうえお", i2.next());
305
+ assertEquals(new BigDecimal("-9999"), i2.next());
306
+ assertEquals(new BigDecimal("-99999999.99"), i2.next());
307
+ assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
308
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
309
+ }
310
+ {
311
+ Iterator<Object> i2 = i1.next().iterator();
312
+ assertEquals("A003", i2.next());
313
+ assertEquals(null, i2.next());
314
+ assertEquals(null, i2.next());
315
+ assertEquals(null, i2.next());
316
+ assertEquals(null, i2.next());
317
+ assertEquals(null, i2.next());
318
+ }
319
+ }
320
+
321
+ private void assertTableEmpty(String table) throws Exception
322
+ {
323
+ List<List<Object>> rows = select(table);
324
+ assertEquals(0, rows.size());
325
+ }
326
+
327
+ private void assertGeneratedTable1(String table) throws Exception
328
+ {
329
+ // datetime of UTC will be inserted by embulk.
330
+ // datetime of default timezone will be selected by JDBC.
331
+ TimeZone timeZone = TimeZone.getDefault();
332
+ List<List<Object>> rows = select(table);
333
+
334
+ /*
335
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
336
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
337
+ A003,,,,,
338
+ */
339
+
340
+ assertEquals(3, rows.size());
341
+ Iterator<List<Object>> i1 = rows.iterator();
342
+ {
343
+ Iterator<Object> i2 = i1.next().iterator();
344
+ assertEquals("A001", i2.next());
345
+ assertEquals("ABCDE", i2.next());
346
+ assertEquals(new BigDecimal("0"), i2.next());
347
+ assertEquals("123.45", i2.next());
348
+ assertEquals(toOracleTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
349
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
350
+ }
351
+ {
352
+ Iterator<Object> i2 = i1.next().iterator();
353
+ assertEquals("A002", i2.next());
354
+ assertEquals("あいうえお", i2.next());
355
+ assertEquals(new BigDecimal("-9999"), i2.next());
356
+ assertEquals("-99999999.99", i2.next());
357
+ assertEquals(toOracleTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
358
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
359
+ }
360
+ {
361
+ Iterator<Object> i2 = i1.next().iterator();
362
+ assertEquals("A003", i2.next());
363
+ assertEquals(null, i2.next());
364
+ assertEquals(null, i2.next());
365
+ assertEquals(null, i2.next());
366
+ assertEquals(null, i2.next());
367
+ assertEquals(null, i2.next());
368
+ }
369
+ }
370
+
371
+ private void assertGeneratedTable2(String table) throws Exception
372
+ {
373
+ // datetime of UTC will be inserted by embulk.
374
+ // datetime of default timezone will be selected by JDBC.
375
+ TimeZone timeZone = TimeZone.getDefault();
376
+ List<List<Object>> rows = select(table);
377
+
378
+ /*
379
+ A001,ABCDE,0,123.45,2015/03/05,2015/03/05 12:34:56
380
+ A002,あいうえお,-9999,-99999999.99,2015/03/06,2015/03/06 23:59:59
381
+ A003,,,,,
382
+ */
383
+
384
+ assertEquals(3, rows.size());
385
+ Iterator<List<Object>> i1 = rows.iterator();
386
+ {
387
+ Iterator<Object> i2 = i1.next().iterator();
388
+ assertEquals("A001", i2.next());
389
+ assertEquals("ABCDE", i2.next());
390
+ assertEquals(new BigDecimal("0"), i2.next());
391
+ assertEquals(new BigDecimal("123.45"), i2.next());
392
+ assertEquals(toTimestamp("2015/03/05 00:00:00", timeZone), i2.next());
393
+ assertEquals(toOracleTimestamp("2015/03/05 12:34:56", timeZone), i2.next());
394
+ }
395
+ {
396
+ Iterator<Object> i2 = i1.next().iterator();
397
+ assertEquals("A002", i2.next());
398
+ assertEquals("あいうえお", i2.next());
399
+ assertEquals(new BigDecimal("-9999"), i2.next());
400
+ assertEquals(new BigDecimal("-99999999.99"), i2.next());
401
+ assertEquals(toTimestamp("2015/03/06 00:00:00", timeZone), i2.next());
402
+ assertEquals(toOracleTimestamp("2015/03/06 23:59:59", timeZone), i2.next());
403
+ }
404
+ {
405
+ Iterator<Object> i2 = i1.next().iterator();
406
+ assertEquals("A003", i2.next());
407
+ assertEquals(null, i2.next());
408
+ assertEquals(null, i2.next());
409
+ assertEquals(null, i2.next());
410
+ assertEquals(null, i2.next());
411
+ assertEquals(null, i2.next());
412
+ }
413
+ }
414
+
415
+
416
+ private Timestamp toTimestamp(String s, TimeZone timeZone)
417
+ {
418
+ for (String formatString : new String[]{"yyyy/MM/dd HH:mm:ss", "yyyy/MM/dd"}) {
419
+ DateFormat dateFormat = new SimpleDateFormat(formatString);
420
+ dateFormat.setTimeZone(timeZone);
421
+ try {
422
+ Date date = dateFormat.parse(s);
423
+ return new Timestamp(date.getTime());
424
+ } catch (ParseException e) {
425
+ // NOP
426
+ }
427
+ }
428
+ throw new IllegalArgumentException(s);
429
+ }
430
+
431
+ private Object toOracleTimestamp(String s, TimeZone timeZone) throws Exception
432
+ {
433
+ Class<?> timestampClass = Class.forName("oracle.sql.TIMESTAMP");
434
+ Constructor<?> constructor = timestampClass.getConstructor(Timestamp.class);
435
+ return constructor.newInstance(toTimestamp(s, timeZone));
436
+ }
437
+
438
+
439
+ private List<List<Object>> select(String table) throws SQLException
440
+ {
441
+ try (Connection connection = connect()) {
442
+ try (Statement statement = connection.createStatement()) {
443
+ List<List<Object>> rows = new ArrayList<List<Object>>();
444
+ String sql = "SELECT * FROM " + table;
445
+ System.out.println(sql);
446
+ try (ResultSet resultSet = statement.executeQuery(sql)) {
447
+ while (resultSet.next()) {
448
+ List<Object> row = new ArrayList<Object>();
449
+ for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
450
+ Object value = resultSet.getObject(i);
451
+ if (value != null && value.getClass().getName().equals("oracle.sql.CLOB")) {
452
+ value = resultSet.getString(i);
453
+ }
454
+ row.add(value);
455
+ }
456
+ rows.add(row);
457
+ }
458
+ }
459
+ // cannot sort by CLOB, so sort by Java
460
+ Collections.sort(rows, new Comparator<List<Object>>() {
461
+ @Override
462
+ public int compare(List<Object> o1, List<Object> o2) {
463
+ return o1.toString().compareTo(o2.toString());
464
+ }
465
+ });
466
+ return rows;
467
+ }
468
+ }
469
+
470
+ }
471
+
472
+
473
+ private void executeSQL(String sql) throws SQLException
474
+ {
475
+ executeSQL(sql, false);
476
+ }
477
+
478
+ private void executeSQL(String sql, boolean ignoreError) throws SQLException
479
+ {
480
+ try (Connection connection = connect()) {
481
+ try {
482
+ connection.setAutoCommit(true);
483
+
484
+ try (Statement statement = connection.createStatement()) {
485
+ System.out.println(String.format("Execute SQL : \"%s\".", sql));
486
+ statement.execute(sql);
487
+ }
488
+
489
+ } catch (SQLException e) {
490
+ if (!ignoreError) {
491
+ throw e;
492
+ }
493
+ }
494
+ }
495
+ }
496
+
497
+ private static Connection connect() throws SQLException
498
+ {
499
+ return DriverManager.getConnection("jdbc:oracle:thin:@localhost:1521:TESTDB", "TEST_USER", "test_pw");
500
+ }
501
+
502
+ private void run(String ymlName) throws Exception
503
+ {
504
+ tester.run(convertYml(ymlName));
505
+ }
506
+
507
+ private String convertYml(String ymlName)
508
+ {
509
+ try {
510
+ File ymlPath = convertPath(ymlName);
511
+ File tempYmlPath = new File(ymlPath.getParentFile(), "temp-" + ymlPath.getName());
512
+ Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
513
+ try (BufferedReader reader = new BufferedReader(new FileReader(ymlPath))) {
514
+ try (BufferedWriter writer = new BufferedWriter(new FileWriter(tempYmlPath))) {
515
+ String line;
516
+ while ((line = reader.readLine()) != null) {
517
+ line = line.replaceAll("type: oracle", "type: " + pluginName);
518
+ Matcher matcher = pathPrefixPattern.matcher(line);
519
+ if (matcher.matches()) {
520
+ int group = 2;
521
+ writer.write(line.substring(0, matcher.start(group)));
522
+ writer.write(convertPath(matcher.group(group)).getAbsolutePath());
523
+ writer.write(line.substring(matcher.end(group)));
524
+ } else {
525
+ writer.write(line);
526
+ }
527
+ writer.newLine();
528
+ }
529
+ }
530
+ }
531
+ return tempYmlPath.getAbsolutePath();
532
+
533
+ } catch (IOException e) {
534
+ throw new RuntimeException(e);
535
+ }
536
+ }
537
+
538
+ private File convertPath(String name)
539
+ {
540
+ try {
541
+ File root = new File(getClass().getResource("/dummy.txt").toURI()).getParentFile();
542
+ return new File(root, name);
543
+ } catch (URISyntaxException e) {
544
+ throw new RuntimeException(e);
545
+ }
546
+ }
547
+
548
+ }