embulk-output-oracle 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (56) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +160 -160
  3. data/build.gradle +6 -6
  4. data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
  5. data/classpath/{embulk-output-oracle-0.4.0.jar → embulk-output-oracle-0.4.1.jar} +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +153 -153
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -165
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
  37. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +124 -120
  38. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +105 -100
  39. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +38 -14
  40. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +540 -484
  41. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  42. data/src/test/resources/data/test1/test1.csv +3 -3
  43. data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -0
  44. data/src/test/resources/yml/test-insert-direct-empty.yml +25 -0
  45. data/src/test/resources/yml/{test-insert-oci-split.yml → test-insert-direct-oci-method-split.yml} +28 -28
  46. data/src/test/resources/yml/{test-insert-oci.yml → test-insert-direct-oci-method.yml} +28 -28
  47. data/src/test/resources/yml/test-insert-direct.yml +25 -26
  48. data/src/test/resources/yml/test-insert-empty.yml +27 -0
  49. data/src/test/resources/yml/test-insert.yml +27 -25
  50. data/src/test/resources/yml/test-replace-empty.yml +31 -0
  51. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
  52. data/src/test/resources/yml/test-replace-long-name.yml +31 -31
  53. data/src/test/resources/yml/test-replace.yml +31 -31
  54. data/src/test/resources/yml/test-string-timestamp.yml +28 -28
  55. data/src/test/resources/yml/test-url.yml +24 -24
  56. metadata +10 -6
@@ -1,153 +1,153 @@
1
- package org.embulk.output;
2
-
3
- import java.util.List;
4
- import java.util.Properties;
5
- import java.io.IOException;
6
- import java.nio.ByteBuffer;
7
- import java.sql.SQLException;
8
- import com.google.common.base.Optional;
9
- import com.google.common.collect.ImmutableSet;
10
- import org.embulk.config.Config;
11
- import org.embulk.config.ConfigDefault;
12
- import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
13
- import org.embulk.output.jdbc.BatchInsert;
14
- import org.embulk.output.jdbc.JdbcOutputConnection;
15
- import org.embulk.output.jdbc.JdbcOutputConnector;
16
- import org.embulk.output.jdbc.StandardBatchInsert;
17
- import org.embulk.output.oracle.DirectBatchInsert;
18
- import org.embulk.output.oracle.InsertMethod;
19
- import org.embulk.output.oracle.OracleCharset;
20
- import org.embulk.output.oracle.OracleOutputConnection;
21
- import org.embulk.output.oracle.OracleOutputConnector;
22
- import org.embulk.spi.PageReader;
23
-
24
- public class OracleOutputPlugin
25
- extends AbstractJdbcOutputPlugin
26
- {
27
- public interface OraclePluginTask
28
- extends PluginTask
29
- {
30
- @Config("driver_path")
31
- @ConfigDefault("null")
32
- public Optional<String> getDriverPath();
33
-
34
- @Config("host")
35
- @ConfigDefault("null")
36
- public Optional<String> getHost();
37
-
38
- @Config("port")
39
- @ConfigDefault("1521")
40
- public int getPort();
41
-
42
- @Config("database")
43
- @ConfigDefault("null")
44
- public Optional<String> getDatabase();
45
-
46
- @Config("url")
47
- @ConfigDefault("null")
48
- public Optional<String> getUrl();
49
-
50
- @Config("user")
51
- public String getUser();
52
-
53
- @Config("password")
54
- @ConfigDefault("\"\"")
55
- public String getPassword();
56
-
57
- @Config("insert_method")
58
- @ConfigDefault("\"normal\"")
59
- public InsertMethod getInsertMethod();
60
- }
61
-
62
- @Override
63
- protected Class<? extends PluginTask> getTaskClass()
64
- {
65
- return OraclePluginTask.class;
66
- }
67
-
68
- @Override
69
- protected Features getFeatures(PluginTask task)
70
- {
71
- return new Features()
72
- .setMaxTableNameLength(30)
73
- .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
74
- .setIgnoreMergeKeys(false);
75
- }
76
-
77
- @Override
78
- protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
79
- {
80
- OraclePluginTask oracleTask = (OraclePluginTask) task;
81
-
82
- if (oracleTask.getDriverPath().isPresent()) {
83
- loadDriverJar(oracleTask.getDriverPath().get());
84
- }
85
-
86
- String url;
87
- if (oracleTask.getInsertMethod() == InsertMethod.oci) {
88
- if (!oracleTask.getHost().isPresent()) {
89
- throw new IllegalArgumentException("Field 'host' is not set.");
90
- }
91
- if (!oracleTask.getDatabase().isPresent()) {
92
- throw new IllegalArgumentException("Field 'database' is not set.");
93
- }
94
- } else {
95
- if (oracleTask.getUrl().isPresent()) {
96
- if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
97
- throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
98
- }
99
- } else {
100
- if (!oracleTask.getHost().isPresent()) {
101
- throw new IllegalArgumentException("Field 'host' is not set.");
102
- }
103
- if (!oracleTask.getDatabase().isPresent()) {
104
- throw new IllegalArgumentException("Field 'database' is not set.");
105
- }
106
- }
107
- }
108
-
109
- if (oracleTask.getUrl().isPresent()) {
110
- url = oracleTask.getUrl().get();
111
- } else {
112
- url = String.format("jdbc:oracle:thin:@%s:%d:%s",
113
- oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
114
- }
115
-
116
- Properties props = new Properties();
117
- props.putAll(oracleTask.getOptions());
118
-
119
- props.setProperty("user", oracleTask.getUser());
120
- logger.info("Connecting to {} options {}", url, props);
121
- props.setProperty("password", oracleTask.getPassword());
122
-
123
- return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
124
- }
125
-
126
- @Override
127
- protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
128
- {
129
- if (mergeKeys.isPresent()) {
130
- throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
131
- }
132
-
133
- OraclePluginTask oracleTask = (OraclePluginTask) task;
134
- JdbcOutputConnector connector = getConnector(task, true);
135
-
136
- if (oracleTask.getInsertMethod() == InsertMethod.oci) {
137
- OracleCharset charset;
138
- try (JdbcOutputConnection connection = connector.connect(true)) {
139
- charset = ((OracleOutputConnection)connection).getOracleCharset();
140
- }
141
-
142
- return new DirectBatchInsert(
143
- String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
144
- oracleTask.getUser(),
145
- oracleTask.getPassword(),
146
- oracleTask.getTable(),
147
- charset,
148
- oracleTask.getBatchSize());
149
- }
150
-
151
- return new StandardBatchInsert(getConnector(task, true), mergeKeys);
152
- }
153
- }
1
+ package org.embulk.output;
2
+
3
+ import java.util.List;
4
+ import java.util.Properties;
5
+ import java.io.IOException;
6
+ import java.nio.ByteBuffer;
7
+ import java.sql.SQLException;
8
+ import com.google.common.base.Optional;
9
+ import com.google.common.collect.ImmutableSet;
10
+ import org.embulk.config.Config;
11
+ import org.embulk.config.ConfigDefault;
12
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
13
+ import org.embulk.output.jdbc.BatchInsert;
14
+ import org.embulk.output.jdbc.JdbcOutputConnection;
15
+ import org.embulk.output.jdbc.JdbcOutputConnector;
16
+ import org.embulk.output.jdbc.StandardBatchInsert;
17
+ import org.embulk.output.oracle.DirectBatchInsert;
18
+ import org.embulk.output.oracle.InsertMethod;
19
+ import org.embulk.output.oracle.OracleCharset;
20
+ import org.embulk.output.oracle.OracleOutputConnection;
21
+ import org.embulk.output.oracle.OracleOutputConnector;
22
+ import org.embulk.spi.PageReader;
23
+
24
+ public class OracleOutputPlugin
25
+ extends AbstractJdbcOutputPlugin
26
+ {
27
+ public interface OraclePluginTask
28
+ extends PluginTask
29
+ {
30
+ @Config("driver_path")
31
+ @ConfigDefault("null")
32
+ public Optional<String> getDriverPath();
33
+
34
+ @Config("host")
35
+ @ConfigDefault("null")
36
+ public Optional<String> getHost();
37
+
38
+ @Config("port")
39
+ @ConfigDefault("1521")
40
+ public int getPort();
41
+
42
+ @Config("database")
43
+ @ConfigDefault("null")
44
+ public Optional<String> getDatabase();
45
+
46
+ @Config("url")
47
+ @ConfigDefault("null")
48
+ public Optional<String> getUrl();
49
+
50
+ @Config("user")
51
+ public String getUser();
52
+
53
+ @Config("password")
54
+ @ConfigDefault("\"\"")
55
+ public String getPassword();
56
+
57
+ @Config("insert_method")
58
+ @ConfigDefault("\"normal\"")
59
+ public InsertMethod getInsertMethod();
60
+ }
61
+
62
+ @Override
63
+ protected Class<? extends PluginTask> getTaskClass()
64
+ {
65
+ return OraclePluginTask.class;
66
+ }
67
+
68
+ @Override
69
+ protected Features getFeatures(PluginTask task)
70
+ {
71
+ return new Features()
72
+ .setMaxTableNameLength(30)
73
+ .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
74
+ .setIgnoreMergeKeys(false);
75
+ }
76
+
77
+ @Override
78
+ protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
79
+ {
80
+ OraclePluginTask oracleTask = (OraclePluginTask) task;
81
+
82
+ if (oracleTask.getDriverPath().isPresent()) {
83
+ loadDriverJar(oracleTask.getDriverPath().get());
84
+ }
85
+
86
+ String url;
87
+ if (oracleTask.getInsertMethod() == InsertMethod.oci) {
88
+ if (!oracleTask.getHost().isPresent()) {
89
+ throw new IllegalArgumentException("Field 'host' is not set.");
90
+ }
91
+ if (!oracleTask.getDatabase().isPresent()) {
92
+ throw new IllegalArgumentException("Field 'database' is not set.");
93
+ }
94
+ } else {
95
+ if (oracleTask.getUrl().isPresent()) {
96
+ if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
97
+ throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
98
+ }
99
+ } else {
100
+ if (!oracleTask.getHost().isPresent()) {
101
+ throw new IllegalArgumentException("Field 'host' is not set.");
102
+ }
103
+ if (!oracleTask.getDatabase().isPresent()) {
104
+ throw new IllegalArgumentException("Field 'database' is not set.");
105
+ }
106
+ }
107
+ }
108
+
109
+ if (oracleTask.getUrl().isPresent()) {
110
+ url = oracleTask.getUrl().get();
111
+ } else {
112
+ url = String.format("jdbc:oracle:thin:@%s:%d:%s",
113
+ oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
114
+ }
115
+
116
+ Properties props = new Properties();
117
+ props.putAll(oracleTask.getOptions());
118
+
119
+ props.setProperty("user", oracleTask.getUser());
120
+ logger.info("Connecting to {} options {}", url, props);
121
+ props.setProperty("password", oracleTask.getPassword());
122
+
123
+ return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
124
+ }
125
+
126
+ @Override
127
+ protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
128
+ {
129
+ if (mergeKeys.isPresent()) {
130
+ throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
131
+ }
132
+
133
+ OraclePluginTask oracleTask = (OraclePluginTask) task;
134
+ JdbcOutputConnector connector = getConnector(task, true);
135
+
136
+ if (oracleTask.getInsertMethod() == InsertMethod.oci) {
137
+ OracleCharset charset;
138
+ try (JdbcOutputConnection connection = connector.connect(true)) {
139
+ charset = ((OracleOutputConnection)connection).getOracleCharset();
140
+ }
141
+
142
+ return new DirectBatchInsert(
143
+ String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
144
+ oracleTask.getUser(),
145
+ oracleTask.getPassword(),
146
+ oracleTask.getTable(),
147
+ charset,
148
+ oracleTask.getBatchSize());
149
+ }
150
+
151
+ return new StandardBatchInsert(getConnector(task, true), mergeKeys);
152
+ }
153
+ }
@@ -1,290 +1,290 @@
1
- package org.embulk.output.oracle;
2
-
3
- import java.io.IOException;
4
- import java.math.BigDecimal;
5
- import java.sql.SQLException;
6
- import java.sql.Types;
7
- import java.text.DateFormat;
8
- import java.text.SimpleDateFormat;
9
- import java.util.ArrayList;
10
- import java.util.Arrays;
11
- import java.util.List;
12
- import java.util.Calendar;
13
- import org.embulk.output.jdbc.BatchInsert;
14
- import org.embulk.output.jdbc.JdbcColumn;
15
- import org.embulk.output.jdbc.JdbcSchema;
16
- import org.embulk.output.oracle.oci.ColumnDefinition;
17
- import org.embulk.output.oracle.oci.OCIManager;
18
- import org.embulk.output.oracle.oci.OCIWrapper;
19
- import org.embulk.output.oracle.oci.RowBuffer;
20
- import org.embulk.output.oracle.oci.TableDefinition;
21
- import org.embulk.spi.time.Timestamp;
22
- import org.embulk.spi.Exec;
23
- import org.slf4j.Logger;
24
-
25
- public class DirectBatchInsert implements BatchInsert
26
- {
27
- private static OCIManager ociManager = new OCIManager();
28
-
29
- private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
30
-
31
- private final List<String> ociKey;
32
- private final String database;
33
- private final String user;
34
- private final String password;
35
- private final String table;
36
- private final OracleCharset charset;
37
- private final int batchSize;
38
- private RowBuffer buffer;
39
- private long totalRows;
40
- private int rowSize;
41
- private int batchWeight;
42
-
43
- private DateFormat[] formats;
44
-
45
-
46
- public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
47
- {
48
- this.database = database;
49
- this.user = user;
50
- this.password = password;
51
- this.table = table;
52
- this.charset = charset;
53
- this.batchSize = batchSize;
54
-
55
- ociKey = Arrays.asList(database, user, table);
56
- }
57
-
58
- @Override
59
- public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
60
- {
61
-
62
- /*
63
- * available mappings
64
- *
65
- * boolean -> unused
66
- * byte -> unused
67
- * short -> unused
68
- * int -> NUMBER
69
- * long -> NUMBER
70
- * BigDecimal -> NUMBER
71
- * String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
72
- * NString -> unused
73
- * bytes -> unused
74
- * SqlDate -> unused
75
- * SqlTime -> unused
76
- * SqlTimeStamp -> TIMESTAMP
77
- *
78
- */
79
-
80
- formats = new DateFormat[insertSchema.getCount()];
81
- List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
82
- java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
83
- for (int i = 0; i < insertSchema.getCount(); i++) {
84
- JdbcColumn insertColumn = insertSchema.getColumn(i);
85
- switch (insertColumn.getSqlType()) {
86
- case Types.CHAR:
87
- case Types.VARCHAR:
88
- case Types.LONGVARCHAR:
89
- case Types.CLOB:
90
- // TODO: CHAR(n CHAR)
91
- columns.add(new ColumnDefinition(insertColumn.getName(),
92
- ColumnDefinition.SQLT_CHR,
93
- insertColumn.getSizeTypeParameter()));
94
- break;
95
-
96
- case Types.DECIMAL:
97
- // sign + size
98
- int size = 1 + insertColumn.getSizeTypeParameter();
99
- if (insertColumn.getSizeTypeParameter() > 0) {
100
- // decimal point
101
- size += 1;
102
- }
103
- columns.add(new ColumnDefinition(insertColumn.getName(),
104
- ColumnDefinition.SQLT_CHR,
105
- size));
106
- break;
107
-
108
- case Types.DATE:
109
- break;
110
-
111
- case Types.TIMESTAMP:
112
- String oracleFormat;
113
- DateFormat javaFormat;
114
- if (insertColumn.getSimpleTypeName().equals("DATE")) {
115
- oracleFormat = "YYYY-MM-DD HH24:MI:SS";
116
- javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
117
- } else {
118
- oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
119
- javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
120
- }
121
- formats[i] = javaFormat;
122
- columns.add(new ColumnDefinition(insertColumn.getName(),
123
- ColumnDefinition.SQLT_CHR,
124
- javaFormat.format(dummy).length(),
125
- oracleFormat));
126
- break;
127
-
128
- default:
129
- throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
130
- }
131
-
132
- }
133
-
134
- rowSize = 0;
135
- for (ColumnDefinition column : columns) {
136
- rowSize += column.columnSize;
137
- }
138
-
139
- TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
140
- ociManager.open(ociKey, database, user, password, tableDefinition);
141
-
142
- buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
143
- }
144
-
145
- @Override
146
- public int getBatchWeight()
147
- {
148
- return batchWeight;
149
- }
150
-
151
- @Override
152
- public void add() throws IOException, SQLException
153
- {
154
- batchWeight += rowSize;
155
- if (buffer.isFull()) {
156
- flush();
157
- }
158
- }
159
-
160
- @Override
161
- public void close() throws IOException, SQLException
162
- {
163
- ociManager.close(ociKey);
164
- }
165
-
166
- @Override
167
- public void flush() throws IOException, SQLException
168
- {
169
- if (buffer.getRowCount() > 0) {
170
- try {
171
- logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
172
-
173
- long startTime = System.currentTimeMillis();
174
-
175
- OCIWrapper oci = ociManager.get(ociKey);
176
- synchronized (oci) {
177
- oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
178
- }
179
-
180
- totalRows += buffer.getRowCount();
181
- double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
182
- logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
183
-
184
- } finally {
185
- buffer.clear();
186
- batchWeight = 0;
187
- }
188
- }
189
- }
190
-
191
- @Override
192
- public void finish() throws IOException, SQLException
193
- {
194
- flush();
195
- }
196
-
197
- @Override
198
- public void setNull(int sqlType) throws IOException, SQLException
199
- {
200
- buffer.addValue("");
201
- }
202
-
203
- @Override
204
- public void setBoolean(boolean v) throws IOException, SQLException
205
- {
206
- throw new SQLException("Unsupported");
207
- }
208
-
209
- @Override
210
- public void setByte(byte v) throws IOException, SQLException
211
- {
212
- throw new SQLException("Unsupported");
213
- }
214
-
215
- @Override
216
- public void setShort(short v) throws IOException, SQLException
217
- {
218
- throw new SQLException("Unsupported");
219
- }
220
-
221
- @Override
222
- public void setInt(int v) throws IOException, SQLException
223
- {
224
- buffer.addValue(v);
225
- }
226
-
227
- @Override
228
- public void setLong(long v) throws IOException, SQLException
229
- {
230
- buffer.addValue(Long.toString(v));
231
- }
232
-
233
- @Override
234
- public void setFloat(float v) throws IOException, SQLException
235
- {
236
- throw new SQLException("Unsupported");
237
- }
238
-
239
- @Override
240
- public void setDouble(double v) throws IOException, SQLException
241
- {
242
- throw new SQLException("Unsupported");
243
- }
244
-
245
- @Override
246
- public void setBigDecimal(BigDecimal v) throws IOException, SQLException
247
- {
248
- buffer.addValue(v);
249
- }
250
-
251
- @Override
252
- public void setString(String v) throws IOException, SQLException
253
- {
254
- buffer.addValue(v);
255
- }
256
-
257
- @Override
258
- public void setNString(String v) throws IOException, SQLException
259
- {
260
- throw new SQLException("Unsupported");
261
- }
262
-
263
- @Override
264
- public void setBytes(byte[] v) throws IOException, SQLException
265
- {
266
- throw new SQLException("Unsupported");
267
- }
268
-
269
- @Override
270
- public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
271
- {
272
- throw new SQLException("Unsupported");
273
- }
274
-
275
- @Override
276
- public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
277
- {
278
- throw new SQLException("Unsupported");
279
- }
280
-
281
- @Override
282
- public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
283
- {
284
- java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
285
- t.setNanos(v.getNano());
286
- DateFormat format = formats[buffer.getCurrentColumn()];
287
- format.setTimeZone(calendar.getTimeZone());
288
- buffer.addValue(format.format(t));
289
- }
290
- }
1
+ package org.embulk.output.oracle;
2
+
3
+ import java.io.IOException;
4
+ import java.math.BigDecimal;
5
+ import java.sql.SQLException;
6
+ import java.sql.Types;
7
+ import java.text.DateFormat;
8
+ import java.text.SimpleDateFormat;
9
+ import java.util.ArrayList;
10
+ import java.util.Arrays;
11
+ import java.util.List;
12
+ import java.util.Calendar;
13
+ import org.embulk.output.jdbc.BatchInsert;
14
+ import org.embulk.output.jdbc.JdbcColumn;
15
+ import org.embulk.output.jdbc.JdbcSchema;
16
+ import org.embulk.output.oracle.oci.ColumnDefinition;
17
+ import org.embulk.output.oracle.oci.OCIManager;
18
+ import org.embulk.output.oracle.oci.OCIWrapper;
19
+ import org.embulk.output.oracle.oci.RowBuffer;
20
+ import org.embulk.output.oracle.oci.TableDefinition;
21
+ import org.embulk.spi.time.Timestamp;
22
+ import org.embulk.spi.Exec;
23
+ import org.slf4j.Logger;
24
+
25
+ public class DirectBatchInsert implements BatchInsert
26
+ {
27
+ private static OCIManager ociManager = new OCIManager();
28
+
29
+ private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
30
+
31
+ private final List<String> ociKey;
32
+ private final String database;
33
+ private final String user;
34
+ private final String password;
35
+ private final String table;
36
+ private final OracleCharset charset;
37
+ private final int batchSize;
38
+ private RowBuffer buffer;
39
+ private long totalRows;
40
+ private int rowSize;
41
+ private int batchWeight;
42
+
43
+ private DateFormat[] formats;
44
+
45
+
46
+ public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
47
+ {
48
+ this.database = database;
49
+ this.user = user;
50
+ this.password = password;
51
+ this.table = table;
52
+ this.charset = charset;
53
+ this.batchSize = batchSize;
54
+
55
+ ociKey = Arrays.asList(database, user, table);
56
+ }
57
+
58
+ @Override
59
+ public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
60
+ {
61
+
62
+ /*
63
+ * available mappings
64
+ *
65
+ * boolean -> unused
66
+ * byte -> unused
67
+ * short -> unused
68
+ * int -> NUMBER
69
+ * long -> NUMBER
70
+ * BigDecimal -> NUMBER
71
+ * String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
72
+ * NString -> unused
73
+ * bytes -> unused
74
+ * SqlDate -> unused
75
+ * SqlTime -> unused
76
+ * SqlTimeStamp -> TIMESTAMP
77
+ *
78
+ */
79
+
80
+ formats = new DateFormat[insertSchema.getCount()];
81
+ List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
82
+ java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
83
+ for (int i = 0; i < insertSchema.getCount(); i++) {
84
+ JdbcColumn insertColumn = insertSchema.getColumn(i);
85
+ switch (insertColumn.getSqlType()) {
86
+ case Types.CHAR:
87
+ case Types.VARCHAR:
88
+ case Types.LONGVARCHAR:
89
+ case Types.CLOB:
90
+ // TODO: CHAR(n CHAR)
91
+ columns.add(new ColumnDefinition(insertColumn.getName(),
92
+ ColumnDefinition.SQLT_CHR,
93
+ insertColumn.getSizeTypeParameter()));
94
+ break;
95
+
96
+ case Types.DECIMAL:
97
+ // sign + size
98
+ int size = 1 + insertColumn.getSizeTypeParameter();
99
+ if (insertColumn.getSizeTypeParameter() > 0) {
100
+ // decimal point
101
+ size += 1;
102
+ }
103
+ columns.add(new ColumnDefinition(insertColumn.getName(),
104
+ ColumnDefinition.SQLT_CHR,
105
+ size));
106
+ break;
107
+
108
+ case Types.DATE:
109
+ break;
110
+
111
+ case Types.TIMESTAMP:
112
+ String oracleFormat;
113
+ DateFormat javaFormat;
114
+ if (insertColumn.getSimpleTypeName().equals("DATE")) {
115
+ oracleFormat = "YYYY-MM-DD HH24:MI:SS";
116
+ javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
117
+ } else {
118
+ oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
119
+ javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
120
+ }
121
+ formats[i] = javaFormat;
122
+ columns.add(new ColumnDefinition(insertColumn.getName(),
123
+ ColumnDefinition.SQLT_CHR,
124
+ javaFormat.format(dummy).length(),
125
+ oracleFormat));
126
+ break;
127
+
128
+ default:
129
+ throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
130
+ }
131
+
132
+ }
133
+
134
+ rowSize = 0;
135
+ for (ColumnDefinition column : columns) {
136
+ rowSize += column.columnSize;
137
+ }
138
+
139
+ TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
140
+ ociManager.open(ociKey, database, user, password, tableDefinition);
141
+
142
+ buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
143
+ }
144
+
145
+ @Override
146
+ public int getBatchWeight()
147
+ {
148
+ return batchWeight;
149
+ }
150
+
151
+ @Override
152
+ public void add() throws IOException, SQLException
153
+ {
154
+ batchWeight += rowSize;
155
+ if (buffer.isFull()) {
156
+ flush();
157
+ }
158
+ }
159
+
160
+ @Override
161
+ public void close() throws IOException, SQLException
162
+ {
163
+ ociManager.close(ociKey);
164
+ }
165
+
166
+ @Override
167
+ public void flush() throws IOException, SQLException
168
+ {
169
+ if (buffer.getRowCount() > 0) {
170
+ try {
171
+ logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
172
+
173
+ long startTime = System.currentTimeMillis();
174
+
175
+ OCIWrapper oci = ociManager.get(ociKey);
176
+ synchronized (oci) {
177
+ oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
178
+ }
179
+
180
+ totalRows += buffer.getRowCount();
181
+ double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
182
+ logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
183
+
184
+ } finally {
185
+ buffer.clear();
186
+ batchWeight = 0;
187
+ }
188
+ }
189
+ }
190
+
191
+ @Override
192
+ public void finish() throws IOException, SQLException
193
+ {
194
+ flush();
195
+ }
196
+
197
+ @Override
198
+ public void setNull(int sqlType) throws IOException, SQLException
199
+ {
200
+ buffer.addValue("");
201
+ }
202
+
203
+ @Override
204
+ public void setBoolean(boolean v) throws IOException, SQLException
205
+ {
206
+ throw new SQLException("Unsupported");
207
+ }
208
+
209
+ @Override
210
+ public void setByte(byte v) throws IOException, SQLException
211
+ {
212
+ throw new SQLException("Unsupported");
213
+ }
214
+
215
+ @Override
216
+ public void setShort(short v) throws IOException, SQLException
217
+ {
218
+ throw new SQLException("Unsupported");
219
+ }
220
+
221
+ @Override
222
+ public void setInt(int v) throws IOException, SQLException
223
+ {
224
+ buffer.addValue(v);
225
+ }
226
+
227
+ @Override
228
+ public void setLong(long v) throws IOException, SQLException
229
+ {
230
+ buffer.addValue(Long.toString(v));
231
+ }
232
+
233
+ @Override
234
+ public void setFloat(float v) throws IOException, SQLException
235
+ {
236
+ throw new SQLException("Unsupported");
237
+ }
238
+
239
+ @Override
240
+ public void setDouble(double v) throws IOException, SQLException
241
+ {
242
+ throw new SQLException("Unsupported");
243
+ }
244
+
245
+ @Override
246
+ public void setBigDecimal(BigDecimal v) throws IOException, SQLException
247
+ {
248
+ buffer.addValue(v);
249
+ }
250
+
251
+ @Override
252
+ public void setString(String v) throws IOException, SQLException
253
+ {
254
+ buffer.addValue(v);
255
+ }
256
+
257
+ @Override
258
+ public void setNString(String v) throws IOException, SQLException
259
+ {
260
+ throw new SQLException("Unsupported");
261
+ }
262
+
263
+ @Override
264
+ public void setBytes(byte[] v) throws IOException, SQLException
265
+ {
266
+ throw new SQLException("Unsupported");
267
+ }
268
+
269
+ @Override
270
+ public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
271
+ {
272
+ throw new SQLException("Unsupported");
273
+ }
274
+
275
+ @Override
276
+ public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
277
+ {
278
+ throw new SQLException("Unsupported");
279
+ }
280
+
281
+ @Override
282
+ public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
283
+ {
284
+ java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
285
+ t.setNanos(v.getNano());
286
+ DateFormat format = formats[buffer.getCurrentColumn()];
287
+ format.setTimeZone(calendar.getTimeZone());
288
+ buffer.addValue(format.format(t));
289
+ }
290
+ }