embulk-output-oracle 0.4.1 → 0.4.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +160 -160
  3. data/build.gradle +6 -6
  4. data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
  5. data/classpath/{embulk-output-oracle-0.4.1.jar → embulk-output-oracle-0.4.2.jar} +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -153
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +179 -165
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +185 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +50 -25
  37. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +548 -540
  38. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  39. data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java +79 -0
  40. data/src/test/resources/data/test1/test1.csv +3 -3
  41. data/src/test/resources/dummy.txt +1 -0
  42. data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -26
  43. data/src/test/resources/yml/test-insert-direct-empty.yml +25 -25
  44. data/src/test/resources/yml/test-insert-direct-oci-method-split.yml +28 -28
  45. data/src/test/resources/yml/test-insert-direct-oci-method.yml +28 -28
  46. data/src/test/resources/yml/test-insert-direct.yml +25 -25
  47. data/src/test/resources/yml/test-insert-empty.yml +27 -27
  48. data/src/test/resources/yml/test-insert.yml +25 -27
  49. data/src/test/resources/yml/test-replace-empty.yml +31 -31
  50. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
  51. data/src/test/resources/yml/test-replace-long-name.yml +31 -31
  52. data/src/test/resources/yml/test-replace.yml +31 -31
  53. data/src/test/resources/yml/test-string-timestamp.yml +28 -28
  54. data/src/test/resources/yml/test-url.yml +24 -24
  55. metadata +6 -8
  56. data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
  57. data/lib/embulk/linux_x64/libembulk-output-oracle.so +0 -0
  58. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +0 -42
  59. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +0 -124
  60. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +0 -105
@@ -1,153 +1,151 @@
1
- package org.embulk.output;
2
-
3
- import java.util.List;
4
- import java.util.Properties;
5
- import java.io.IOException;
6
- import java.nio.ByteBuffer;
7
- import java.sql.SQLException;
8
- import com.google.common.base.Optional;
9
- import com.google.common.collect.ImmutableSet;
10
- import org.embulk.config.Config;
11
- import org.embulk.config.ConfigDefault;
12
- import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
13
- import org.embulk.output.jdbc.BatchInsert;
14
- import org.embulk.output.jdbc.JdbcOutputConnection;
15
- import org.embulk.output.jdbc.JdbcOutputConnector;
16
- import org.embulk.output.jdbc.StandardBatchInsert;
17
- import org.embulk.output.oracle.DirectBatchInsert;
18
- import org.embulk.output.oracle.InsertMethod;
19
- import org.embulk.output.oracle.OracleCharset;
20
- import org.embulk.output.oracle.OracleOutputConnection;
21
- import org.embulk.output.oracle.OracleOutputConnector;
22
- import org.embulk.spi.PageReader;
23
-
24
- public class OracleOutputPlugin
25
- extends AbstractJdbcOutputPlugin
26
- {
27
- public interface OraclePluginTask
28
- extends PluginTask
29
- {
30
- @Config("driver_path")
31
- @ConfigDefault("null")
32
- public Optional<String> getDriverPath();
33
-
34
- @Config("host")
35
- @ConfigDefault("null")
36
- public Optional<String> getHost();
37
-
38
- @Config("port")
39
- @ConfigDefault("1521")
40
- public int getPort();
41
-
42
- @Config("database")
43
- @ConfigDefault("null")
44
- public Optional<String> getDatabase();
45
-
46
- @Config("url")
47
- @ConfigDefault("null")
48
- public Optional<String> getUrl();
49
-
50
- @Config("user")
51
- public String getUser();
52
-
53
- @Config("password")
54
- @ConfigDefault("\"\"")
55
- public String getPassword();
56
-
57
- @Config("insert_method")
58
- @ConfigDefault("\"normal\"")
59
- public InsertMethod getInsertMethod();
60
- }
61
-
62
- @Override
63
- protected Class<? extends PluginTask> getTaskClass()
64
- {
65
- return OraclePluginTask.class;
66
- }
67
-
68
- @Override
69
- protected Features getFeatures(PluginTask task)
70
- {
71
- return new Features()
72
- .setMaxTableNameLength(30)
73
- .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
74
- .setIgnoreMergeKeys(false);
75
- }
76
-
77
- @Override
78
- protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
79
- {
80
- OraclePluginTask oracleTask = (OraclePluginTask) task;
81
-
82
- if (oracleTask.getDriverPath().isPresent()) {
83
- loadDriverJar(oracleTask.getDriverPath().get());
84
- }
85
-
86
- String url;
87
- if (oracleTask.getInsertMethod() == InsertMethod.oci) {
88
- if (!oracleTask.getHost().isPresent()) {
89
- throw new IllegalArgumentException("Field 'host' is not set.");
90
- }
91
- if (!oracleTask.getDatabase().isPresent()) {
92
- throw new IllegalArgumentException("Field 'database' is not set.");
93
- }
94
- } else {
95
- if (oracleTask.getUrl().isPresent()) {
96
- if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
97
- throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
98
- }
99
- } else {
100
- if (!oracleTask.getHost().isPresent()) {
101
- throw new IllegalArgumentException("Field 'host' is not set.");
102
- }
103
- if (!oracleTask.getDatabase().isPresent()) {
104
- throw new IllegalArgumentException("Field 'database' is not set.");
105
- }
106
- }
107
- }
108
-
109
- if (oracleTask.getUrl().isPresent()) {
110
- url = oracleTask.getUrl().get();
111
- } else {
112
- url = String.format("jdbc:oracle:thin:@%s:%d:%s",
113
- oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
114
- }
115
-
116
- Properties props = new Properties();
117
- props.putAll(oracleTask.getOptions());
118
-
119
- props.setProperty("user", oracleTask.getUser());
120
- logger.info("Connecting to {} options {}", url, props);
121
- props.setProperty("password", oracleTask.getPassword());
122
-
123
- return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
124
- }
125
-
126
- @Override
127
- protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
128
- {
129
- if (mergeKeys.isPresent()) {
130
- throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
131
- }
132
-
133
- OraclePluginTask oracleTask = (OraclePluginTask) task;
134
- JdbcOutputConnector connector = getConnector(task, true);
135
-
136
- if (oracleTask.getInsertMethod() == InsertMethod.oci) {
137
- OracleCharset charset;
138
- try (JdbcOutputConnection connection = connector.connect(true)) {
139
- charset = ((OracleOutputConnection)connection).getOracleCharset();
140
- }
141
-
142
- return new DirectBatchInsert(
143
- String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
144
- oracleTask.getUser(),
145
- oracleTask.getPassword(),
146
- oracleTask.getTable(),
147
- charset,
148
- oracleTask.getBatchSize());
149
- }
150
-
151
- return new StandardBatchInsert(getConnector(task, true), mergeKeys);
152
- }
153
- }
1
+ package org.embulk.output;
2
+
3
+ import java.util.List;
4
+ import java.util.Properties;
5
+ import java.io.IOException;
6
+ import java.sql.SQLException;
7
+ import com.google.common.base.Optional;
8
+ import com.google.common.collect.ImmutableSet;
9
+ import org.embulk.config.Config;
10
+ import org.embulk.config.ConfigDefault;
11
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
12
+ import org.embulk.output.jdbc.BatchInsert;
13
+ import org.embulk.output.jdbc.JdbcOutputConnection;
14
+ import org.embulk.output.jdbc.JdbcOutputConnector;
15
+ import org.embulk.output.jdbc.StandardBatchInsert;
16
+ import org.embulk.output.oracle.DirectBatchInsert;
17
+ import org.embulk.output.oracle.InsertMethod;
18
+ import org.embulk.output.oracle.OracleCharset;
19
+ import org.embulk.output.oracle.OracleOutputConnection;
20
+ import org.embulk.output.oracle.OracleOutputConnector;
21
+
22
+ public class OracleOutputPlugin
23
+ extends AbstractJdbcOutputPlugin
24
+ {
25
+ public interface OraclePluginTask
26
+ extends PluginTask
27
+ {
28
+ @Config("driver_path")
29
+ @ConfigDefault("null")
30
+ public Optional<String> getDriverPath();
31
+
32
+ @Config("host")
33
+ @ConfigDefault("null")
34
+ public Optional<String> getHost();
35
+
36
+ @Config("port")
37
+ @ConfigDefault("1521")
38
+ public int getPort();
39
+
40
+ @Config("database")
41
+ @ConfigDefault("null")
42
+ public Optional<String> getDatabase();
43
+
44
+ @Config("url")
45
+ @ConfigDefault("null")
46
+ public Optional<String> getUrl();
47
+
48
+ @Config("user")
49
+ public String getUser();
50
+
51
+ @Config("password")
52
+ @ConfigDefault("\"\"")
53
+ public String getPassword();
54
+
55
+ @Config("insert_method")
56
+ @ConfigDefault("\"normal\"")
57
+ public InsertMethod getInsertMethod();
58
+ }
59
+
60
+ @Override
61
+ protected Class<? extends PluginTask> getTaskClass()
62
+ {
63
+ return OraclePluginTask.class;
64
+ }
65
+
66
+ @Override
67
+ protected Features getFeatures(PluginTask task)
68
+ {
69
+ return new Features()
70
+ .setMaxTableNameLength(30)
71
+ .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
72
+ .setIgnoreMergeKeys(false);
73
+ }
74
+
75
+ @Override
76
+ protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
77
+ {
78
+ OraclePluginTask oracleTask = (OraclePluginTask) task;
79
+
80
+ if (oracleTask.getDriverPath().isPresent()) {
81
+ loadDriverJar(oracleTask.getDriverPath().get());
82
+ }
83
+
84
+ String url;
85
+ if (oracleTask.getInsertMethod() == InsertMethod.oci) {
86
+ if (!oracleTask.getHost().isPresent()) {
87
+ throw new IllegalArgumentException("Field 'host' is not set.");
88
+ }
89
+ if (!oracleTask.getDatabase().isPresent()) {
90
+ throw new IllegalArgumentException("Field 'database' is not set.");
91
+ }
92
+ } else {
93
+ if (oracleTask.getUrl().isPresent()) {
94
+ if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
95
+ throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
96
+ }
97
+ } else {
98
+ if (!oracleTask.getHost().isPresent()) {
99
+ throw new IllegalArgumentException("Field 'host' is not set.");
100
+ }
101
+ if (!oracleTask.getDatabase().isPresent()) {
102
+ throw new IllegalArgumentException("Field 'database' is not set.");
103
+ }
104
+ }
105
+ }
106
+
107
+ if (oracleTask.getUrl().isPresent()) {
108
+ url = oracleTask.getUrl().get();
109
+ } else {
110
+ url = String.format("jdbc:oracle:thin:@%s:%d:%s",
111
+ oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
112
+ }
113
+
114
+ Properties props = new Properties();
115
+ props.putAll(oracleTask.getOptions());
116
+
117
+ props.setProperty("user", oracleTask.getUser());
118
+ logger.info("Connecting to {} options {}", url, props);
119
+ props.setProperty("password", oracleTask.getPassword());
120
+
121
+ return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
122
+ }
123
+
124
+ @Override
125
+ protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
126
+ {
127
+ if (mergeKeys.isPresent()) {
128
+ throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
129
+ }
130
+
131
+ OraclePluginTask oracleTask = (OraclePluginTask) task;
132
+ JdbcOutputConnector connector = getConnector(task, true);
133
+
134
+ if (oracleTask.getInsertMethod() == InsertMethod.oci) {
135
+ OracleCharset charset;
136
+ try (JdbcOutputConnection connection = connector.connect(true)) {
137
+ charset = ((OracleOutputConnection)connection).getOracleCharset();
138
+ }
139
+
140
+ return new DirectBatchInsert(
141
+ String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
142
+ oracleTask.getUser(),
143
+ oracleTask.getPassword(),
144
+ oracleTask.getTable(),
145
+ charset,
146
+ oracleTask.getBatchSize());
147
+ }
148
+
149
+ return new StandardBatchInsert(getConnector(task, true), mergeKeys);
150
+ }
151
+ }
@@ -1,290 +1,290 @@
1
- package org.embulk.output.oracle;
2
-
3
- import java.io.IOException;
4
- import java.math.BigDecimal;
5
- import java.sql.SQLException;
6
- import java.sql.Types;
7
- import java.text.DateFormat;
8
- import java.text.SimpleDateFormat;
9
- import java.util.ArrayList;
10
- import java.util.Arrays;
11
- import java.util.List;
12
- import java.util.Calendar;
13
- import org.embulk.output.jdbc.BatchInsert;
14
- import org.embulk.output.jdbc.JdbcColumn;
15
- import org.embulk.output.jdbc.JdbcSchema;
16
- import org.embulk.output.oracle.oci.ColumnDefinition;
17
- import org.embulk.output.oracle.oci.OCIManager;
18
- import org.embulk.output.oracle.oci.OCIWrapper;
19
- import org.embulk.output.oracle.oci.RowBuffer;
20
- import org.embulk.output.oracle.oci.TableDefinition;
21
- import org.embulk.spi.time.Timestamp;
22
- import org.embulk.spi.Exec;
23
- import org.slf4j.Logger;
24
-
25
- public class DirectBatchInsert implements BatchInsert
26
- {
27
- private static OCIManager ociManager = new OCIManager();
28
-
29
- private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
30
-
31
- private final List<String> ociKey;
32
- private final String database;
33
- private final String user;
34
- private final String password;
35
- private final String table;
36
- private final OracleCharset charset;
37
- private final int batchSize;
38
- private RowBuffer buffer;
39
- private long totalRows;
40
- private int rowSize;
41
- private int batchWeight;
42
-
43
- private DateFormat[] formats;
44
-
45
-
46
- public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
47
- {
48
- this.database = database;
49
- this.user = user;
50
- this.password = password;
51
- this.table = table;
52
- this.charset = charset;
53
- this.batchSize = batchSize;
54
-
55
- ociKey = Arrays.asList(database, user, table);
56
- }
57
-
58
- @Override
59
- public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
60
- {
61
-
62
- /*
63
- * available mappings
64
- *
65
- * boolean -> unused
66
- * byte -> unused
67
- * short -> unused
68
- * int -> NUMBER
69
- * long -> NUMBER
70
- * BigDecimal -> NUMBER
71
- * String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
72
- * NString -> unused
73
- * bytes -> unused
74
- * SqlDate -> unused
75
- * SqlTime -> unused
76
- * SqlTimeStamp -> TIMESTAMP
77
- *
78
- */
79
-
80
- formats = new DateFormat[insertSchema.getCount()];
81
- List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
82
- java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
83
- for (int i = 0; i < insertSchema.getCount(); i++) {
84
- JdbcColumn insertColumn = insertSchema.getColumn(i);
85
- switch (insertColumn.getSqlType()) {
86
- case Types.CHAR:
87
- case Types.VARCHAR:
88
- case Types.LONGVARCHAR:
89
- case Types.CLOB:
90
- // TODO: CHAR(n CHAR)
91
- columns.add(new ColumnDefinition(insertColumn.getName(),
92
- ColumnDefinition.SQLT_CHR,
93
- insertColumn.getSizeTypeParameter()));
94
- break;
95
-
96
- case Types.DECIMAL:
97
- // sign + size
98
- int size = 1 + insertColumn.getSizeTypeParameter();
99
- if (insertColumn.getSizeTypeParameter() > 0) {
100
- // decimal point
101
- size += 1;
102
- }
103
- columns.add(new ColumnDefinition(insertColumn.getName(),
104
- ColumnDefinition.SQLT_CHR,
105
- size));
106
- break;
107
-
108
- case Types.DATE:
109
- break;
110
-
111
- case Types.TIMESTAMP:
112
- String oracleFormat;
113
- DateFormat javaFormat;
114
- if (insertColumn.getSimpleTypeName().equals("DATE")) {
115
- oracleFormat = "YYYY-MM-DD HH24:MI:SS";
116
- javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
117
- } else {
118
- oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
119
- javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
120
- }
121
- formats[i] = javaFormat;
122
- columns.add(new ColumnDefinition(insertColumn.getName(),
123
- ColumnDefinition.SQLT_CHR,
124
- javaFormat.format(dummy).length(),
125
- oracleFormat));
126
- break;
127
-
128
- default:
129
- throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
130
- }
131
-
132
- }
133
-
134
- rowSize = 0;
135
- for (ColumnDefinition column : columns) {
136
- rowSize += column.columnSize;
137
- }
138
-
139
- TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
140
- ociManager.open(ociKey, database, user, password, tableDefinition);
141
-
142
- buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
143
- }
144
-
145
- @Override
146
- public int getBatchWeight()
147
- {
148
- return batchWeight;
149
- }
150
-
151
- @Override
152
- public void add() throws IOException, SQLException
153
- {
154
- batchWeight += rowSize;
155
- if (buffer.isFull()) {
156
- flush();
157
- }
158
- }
159
-
160
- @Override
161
- public void close() throws IOException, SQLException
162
- {
163
- ociManager.close(ociKey);
164
- }
165
-
166
- @Override
167
- public void flush() throws IOException, SQLException
168
- {
169
- if (buffer.getRowCount() > 0) {
170
- try {
171
- logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
172
-
173
- long startTime = System.currentTimeMillis();
174
-
175
- OCIWrapper oci = ociManager.get(ociKey);
176
- synchronized (oci) {
177
- oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
178
- }
179
-
180
- totalRows += buffer.getRowCount();
181
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
182
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
183
-
184
- } finally {
185
- buffer.clear();
186
- batchWeight = 0;
187
- }
188
- }
189
- }
190
-
191
- @Override
192
- public void finish() throws IOException, SQLException
193
- {
194
- flush();
195
- }
196
-
197
- @Override
198
- public void setNull(int sqlType) throws IOException, SQLException
199
- {
200
- buffer.addValue("");
201
- }
202
-
203
- @Override
204
- public void setBoolean(boolean v) throws IOException, SQLException
205
- {
206
- throw new SQLException("Unsupported");
207
- }
208
-
209
- @Override
210
- public void setByte(byte v) throws IOException, SQLException
211
- {
212
- throw new SQLException("Unsupported");
213
- }
214
-
215
- @Override
216
- public void setShort(short v) throws IOException, SQLException
217
- {
218
- throw new SQLException("Unsupported");
219
- }
220
-
221
- @Override
222
- public void setInt(int v) throws IOException, SQLException
223
- {
224
- buffer.addValue(v);
225
- }
226
-
227
- @Override
228
- public void setLong(long v) throws IOException, SQLException
229
- {
230
- buffer.addValue(Long.toString(v));
231
- }
232
-
233
- @Override
234
- public void setFloat(float v) throws IOException, SQLException
235
- {
236
- throw new SQLException("Unsupported");
237
- }
238
-
239
- @Override
240
- public void setDouble(double v) throws IOException, SQLException
241
- {
242
- throw new SQLException("Unsupported");
243
- }
244
-
245
- @Override
246
- public void setBigDecimal(BigDecimal v) throws IOException, SQLException
247
- {
248
- buffer.addValue(v);
249
- }
250
-
251
- @Override
252
- public void setString(String v) throws IOException, SQLException
253
- {
254
- buffer.addValue(v);
255
- }
256
-
257
- @Override
258
- public void setNString(String v) throws IOException, SQLException
259
- {
260
- throw new SQLException("Unsupported");
261
- }
262
-
263
- @Override
264
- public void setBytes(byte[] v) throws IOException, SQLException
265
- {
266
- throw new SQLException("Unsupported");
267
- }
268
-
269
- @Override
270
- public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
271
- {
272
- throw new SQLException("Unsupported");
273
- }
274
-
275
- @Override
276
- public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
277
- {
278
- throw new SQLException("Unsupported");
279
- }
280
-
281
- @Override
282
- public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
283
- {
284
- java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
285
- t.setNanos(v.getNano());
286
- DateFormat format = formats[buffer.getCurrentColumn()];
287
- format.setTimeZone(calendar.getTimeZone());
288
- buffer.addValue(format.format(t));
289
- }
290
- }
1
+ package org.embulk.output.oracle;
2
+
3
+ import java.io.IOException;
4
+ import java.math.BigDecimal;
5
+ import java.sql.SQLException;
6
+ import java.sql.Types;
7
+ import java.text.DateFormat;
8
+ import java.text.SimpleDateFormat;
9
+ import java.util.ArrayList;
10
+ import java.util.Arrays;
11
+ import java.util.List;
12
+ import java.util.Calendar;
13
+ import org.embulk.output.jdbc.BatchInsert;
14
+ import org.embulk.output.jdbc.JdbcColumn;
15
+ import org.embulk.output.jdbc.JdbcSchema;
16
+ import org.embulk.output.oracle.oci.ColumnDefinition;
17
+ import org.embulk.output.oracle.oci.OCIManager;
18
+ import org.embulk.output.oracle.oci.OCIWrapper;
19
+ import org.embulk.output.oracle.oci.RowBuffer;
20
+ import org.embulk.output.oracle.oci.TableDefinition;
21
+ import org.embulk.spi.time.Timestamp;
22
+ import org.embulk.spi.Exec;
23
+ import org.slf4j.Logger;
24
+
25
+ public class DirectBatchInsert implements BatchInsert
26
+ {
27
+ private static OCIManager ociManager = new OCIManager();
28
+
29
+ private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
30
+
31
+ private final List<String> ociKey;
32
+ private final String database;
33
+ private final String user;
34
+ private final String password;
35
+ private final String table;
36
+ private final OracleCharset charset;
37
+ private final int batchSize;
38
+ private RowBuffer buffer;
39
+ private long totalRows;
40
+ private int rowSize;
41
+ private int batchWeight;
42
+
43
+ private DateFormat[] formats;
44
+
45
+
46
+ public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
47
+ {
48
+ this.database = database;
49
+ this.user = user;
50
+ this.password = password;
51
+ this.table = table;
52
+ this.charset = charset;
53
+ this.batchSize = batchSize;
54
+
55
+ ociKey = Arrays.asList(database, user, table);
56
+ }
57
+
58
+ @Override
59
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
60
+ {
61
+
62
+ /*
63
+ * available mappings
64
+ *
65
+ * boolean -> unused
66
+ * byte -> unused
67
+ * short -> unused
68
+ * int -> NUMBER
69
+ * long -> NUMBER
70
+ * BigDecimal -> NUMBER
71
+ * String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
72
+ * NString -> unused
73
+ * bytes -> unused
74
+ * SqlDate -> unused
75
+ * SqlTime -> unused
76
+ * SqlTimeStamp -> TIMESTAMP
77
+ *
78
+ */
79
+
80
+ formats = new DateFormat[insertSchema.getCount()];
81
+ List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
82
+ java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
83
+ for (int i = 0; i < insertSchema.getCount(); i++) {
84
+ JdbcColumn insertColumn = insertSchema.getColumn(i);
85
+ switch (insertColumn.getSqlType()) {
86
+ case Types.CHAR:
87
+ case Types.VARCHAR:
88
+ case Types.LONGVARCHAR:
89
+ case Types.CLOB:
90
+ // TODO: CHAR(n CHAR)
91
+ columns.add(new ColumnDefinition(insertColumn.getName(),
92
+ ColumnDefinition.SQLT_CHR,
93
+ insertColumn.getSizeTypeParameter()));
94
+ break;
95
+
96
+ case Types.DECIMAL:
97
+ // sign + size
98
+ int size = 1 + insertColumn.getSizeTypeParameter();
99
+ if (insertColumn.getSizeTypeParameter() > 0) {
100
+ // decimal point
101
+ size += 1;
102
+ }
103
+ columns.add(new ColumnDefinition(insertColumn.getName(),
104
+ ColumnDefinition.SQLT_CHR,
105
+ size));
106
+ break;
107
+
108
+ case Types.DATE:
109
+ break;
110
+
111
+ case Types.TIMESTAMP:
112
+ String oracleFormat;
113
+ DateFormat javaFormat;
114
+ if (insertColumn.getSimpleTypeName().equals("DATE")) {
115
+ oracleFormat = "YYYY-MM-DD HH24:MI:SS";
116
+ javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
117
+ } else {
118
+ oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
119
+ javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
120
+ }
121
+ formats[i] = javaFormat;
122
+ columns.add(new ColumnDefinition(insertColumn.getName(),
123
+ ColumnDefinition.SQLT_CHR,
124
+ javaFormat.format(dummy).length(),
125
+ oracleFormat));
126
+ break;
127
+
128
+ default:
129
+ throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
130
+ }
131
+
132
+ }
133
+
134
+ rowSize = 0;
135
+ for (ColumnDefinition column : columns) {
136
+ rowSize += column.columnSize;
137
+ }
138
+
139
+ TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
140
+ ociManager.open(ociKey, database, user, password, tableDefinition);
141
+
142
+ buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
143
+ }
144
+
145
+ @Override
146
+ public int getBatchWeight()
147
+ {
148
+ return batchWeight;
149
+ }
150
+
151
+ @Override
152
+ public void add() throws IOException, SQLException
153
+ {
154
+ batchWeight += rowSize;
155
+ if (buffer.isFull()) {
156
+ flush();
157
+ }
158
+ }
159
+
160
+ @Override
161
+ public void close() throws IOException, SQLException
162
+ {
163
+ ociManager.close(ociKey);
164
+ }
165
+
166
+ @Override
167
+ public void flush() throws IOException, SQLException
168
+ {
169
+ if (buffer.getRowCount() > 0) {
170
+ try {
171
+ logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
172
+
173
+ long startTime = System.currentTimeMillis();
174
+
175
+ OCIWrapper oci = ociManager.get(ociKey);
176
+ synchronized (oci) {
177
+ oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
178
+ }
179
+
180
+ totalRows += buffer.getRowCount();
181
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
182
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
183
+
184
+ } finally {
185
+ buffer.clear();
186
+ batchWeight = 0;
187
+ }
188
+ }
189
+ }
190
+
191
+ @Override
192
+ public void finish() throws IOException, SQLException
193
+ {
194
+ flush();
195
+ }
196
+
197
+ @Override
198
+ public void setNull(int sqlType) throws IOException, SQLException
199
+ {
200
+ buffer.addValue("");
201
+ }
202
+
203
+ @Override
204
+ public void setBoolean(boolean v) throws IOException, SQLException
205
+ {
206
+ throw new SQLException("Unsupported");
207
+ }
208
+
209
+ @Override
210
+ public void setByte(byte v) throws IOException, SQLException
211
+ {
212
+ throw new SQLException("Unsupported");
213
+ }
214
+
215
+ @Override
216
+ public void setShort(short v) throws IOException, SQLException
217
+ {
218
+ throw new SQLException("Unsupported");
219
+ }
220
+
221
+ @Override
222
+ public void setInt(int v) throws IOException, SQLException
223
+ {
224
+ buffer.addValue(v);
225
+ }
226
+
227
+ @Override
228
+ public void setLong(long v) throws IOException, SQLException
229
+ {
230
+ buffer.addValue(Long.toString(v));
231
+ }
232
+
233
+ @Override
234
+ public void setFloat(float v) throws IOException, SQLException
235
+ {
236
+ throw new SQLException("Unsupported");
237
+ }
238
+
239
+ @Override
240
+ public void setDouble(double v) throws IOException, SQLException
241
+ {
242
+ throw new SQLException("Unsupported");
243
+ }
244
+
245
+ @Override
246
+ public void setBigDecimal(BigDecimal v) throws IOException, SQLException
247
+ {
248
+ buffer.addValue(v);
249
+ }
250
+
251
+ @Override
252
+ public void setString(String v) throws IOException, SQLException
253
+ {
254
+ buffer.addValue(v);
255
+ }
256
+
257
+ @Override
258
+ public void setNString(String v) throws IOException, SQLException
259
+ {
260
+ throw new SQLException("Unsupported");
261
+ }
262
+
263
+ @Override
264
+ public void setBytes(byte[] v) throws IOException, SQLException
265
+ {
266
+ throw new SQLException("Unsupported");
267
+ }
268
+
269
+ @Override
270
+ public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
271
+ {
272
+ throw new SQLException("Unsupported");
273
+ }
274
+
275
+ @Override
276
+ public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
277
+ {
278
+ throw new SQLException("Unsupported");
279
+ }
280
+
281
+ @Override
282
+ public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
283
+ {
284
+ java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
285
+ t.setNanos(v.getNano());
286
+ DateFormat format = formats[buffer.getCurrentColumn()];
287
+ format.setTimeZone(calendar.getTimeZone());
288
+ buffer.addValue(format.format(t));
289
+ }
290
+ }