embulk-output-oracle 0.4.1 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +160 -160
- data/build.gradle +6 -6
- data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
- data/classpath/{embulk-output-oracle-0.4.1.jar → embulk-output-oracle-0.4.2.jar} +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -153
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +179 -165
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +185 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +50 -25
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +548 -540
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java +79 -0
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/dummy.txt +1 -0
- data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -26
- data/src/test/resources/yml/test-insert-direct-empty.yml +25 -25
- data/src/test/resources/yml/test-insert-direct-oci-method-split.yml +28 -28
- data/src/test/resources/yml/test-insert-direct-oci-method.yml +28 -28
- data/src/test/resources/yml/test-insert-direct.yml +25 -25
- data/src/test/resources/yml/test-insert-empty.yml +27 -27
- data/src/test/resources/yml/test-insert.yml +25 -27
- data/src/test/resources/yml/test-replace-empty.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name.yml +31 -31
- data/src/test/resources/yml/test-replace.yml +31 -31
- data/src/test/resources/yml/test-string-timestamp.yml +28 -28
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +6 -8
- data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
- data/lib/embulk/linux_x64/libembulk-output-oracle.so +0 -0
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +0 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +0 -124
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +0 -105
@@ -1,153 +1,151 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import java.util.Properties;
|
5
|
-
import java.io.IOException;
|
6
|
-
import java.
|
7
|
-
import
|
8
|
-
import com.google.common.
|
9
|
-
import
|
10
|
-
import org.embulk.config.
|
11
|
-
import org.embulk.
|
12
|
-
import org.embulk.output.jdbc.
|
13
|
-
import org.embulk.output.jdbc.
|
14
|
-
import org.embulk.output.jdbc.
|
15
|
-
import org.embulk.output.jdbc.
|
16
|
-
import org.embulk.output.
|
17
|
-
import org.embulk.output.oracle.
|
18
|
-
import org.embulk.output.oracle.
|
19
|
-
import org.embulk.output.oracle.
|
20
|
-
import org.embulk.output.oracle.
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
.
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
url =
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
props.
|
118
|
-
|
119
|
-
props.setProperty("
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
oracleTask.
|
145
|
-
|
146
|
-
oracleTask.
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
}
|
153
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Properties;
|
5
|
+
import java.io.IOException;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import com.google.common.base.Optional;
|
8
|
+
import com.google.common.collect.ImmutableSet;
|
9
|
+
import org.embulk.config.Config;
|
10
|
+
import org.embulk.config.ConfigDefault;
|
11
|
+
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
12
|
+
import org.embulk.output.jdbc.BatchInsert;
|
13
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
14
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
15
|
+
import org.embulk.output.jdbc.StandardBatchInsert;
|
16
|
+
import org.embulk.output.oracle.DirectBatchInsert;
|
17
|
+
import org.embulk.output.oracle.InsertMethod;
|
18
|
+
import org.embulk.output.oracle.OracleCharset;
|
19
|
+
import org.embulk.output.oracle.OracleOutputConnection;
|
20
|
+
import org.embulk.output.oracle.OracleOutputConnector;
|
21
|
+
|
22
|
+
public class OracleOutputPlugin
|
23
|
+
extends AbstractJdbcOutputPlugin
|
24
|
+
{
|
25
|
+
public interface OraclePluginTask
|
26
|
+
extends PluginTask
|
27
|
+
{
|
28
|
+
@Config("driver_path")
|
29
|
+
@ConfigDefault("null")
|
30
|
+
public Optional<String> getDriverPath();
|
31
|
+
|
32
|
+
@Config("host")
|
33
|
+
@ConfigDefault("null")
|
34
|
+
public Optional<String> getHost();
|
35
|
+
|
36
|
+
@Config("port")
|
37
|
+
@ConfigDefault("1521")
|
38
|
+
public int getPort();
|
39
|
+
|
40
|
+
@Config("database")
|
41
|
+
@ConfigDefault("null")
|
42
|
+
public Optional<String> getDatabase();
|
43
|
+
|
44
|
+
@Config("url")
|
45
|
+
@ConfigDefault("null")
|
46
|
+
public Optional<String> getUrl();
|
47
|
+
|
48
|
+
@Config("user")
|
49
|
+
public String getUser();
|
50
|
+
|
51
|
+
@Config("password")
|
52
|
+
@ConfigDefault("\"\"")
|
53
|
+
public String getPassword();
|
54
|
+
|
55
|
+
@Config("insert_method")
|
56
|
+
@ConfigDefault("\"normal\"")
|
57
|
+
public InsertMethod getInsertMethod();
|
58
|
+
}
|
59
|
+
|
60
|
+
@Override
|
61
|
+
protected Class<? extends PluginTask> getTaskClass()
|
62
|
+
{
|
63
|
+
return OraclePluginTask.class;
|
64
|
+
}
|
65
|
+
|
66
|
+
@Override
|
67
|
+
protected Features getFeatures(PluginTask task)
|
68
|
+
{
|
69
|
+
return new Features()
|
70
|
+
.setMaxTableNameLength(30)
|
71
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
72
|
+
.setIgnoreMergeKeys(false);
|
73
|
+
}
|
74
|
+
|
75
|
+
@Override
|
76
|
+
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
77
|
+
{
|
78
|
+
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
79
|
+
|
80
|
+
if (oracleTask.getDriverPath().isPresent()) {
|
81
|
+
loadDriverJar(oracleTask.getDriverPath().get());
|
82
|
+
}
|
83
|
+
|
84
|
+
String url;
|
85
|
+
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
86
|
+
if (!oracleTask.getHost().isPresent()) {
|
87
|
+
throw new IllegalArgumentException("Field 'host' is not set.");
|
88
|
+
}
|
89
|
+
if (!oracleTask.getDatabase().isPresent()) {
|
90
|
+
throw new IllegalArgumentException("Field 'database' is not set.");
|
91
|
+
}
|
92
|
+
} else {
|
93
|
+
if (oracleTask.getUrl().isPresent()) {
|
94
|
+
if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
|
95
|
+
throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
|
96
|
+
}
|
97
|
+
} else {
|
98
|
+
if (!oracleTask.getHost().isPresent()) {
|
99
|
+
throw new IllegalArgumentException("Field 'host' is not set.");
|
100
|
+
}
|
101
|
+
if (!oracleTask.getDatabase().isPresent()) {
|
102
|
+
throw new IllegalArgumentException("Field 'database' is not set.");
|
103
|
+
}
|
104
|
+
}
|
105
|
+
}
|
106
|
+
|
107
|
+
if (oracleTask.getUrl().isPresent()) {
|
108
|
+
url = oracleTask.getUrl().get();
|
109
|
+
} else {
|
110
|
+
url = String.format("jdbc:oracle:thin:@%s:%d:%s",
|
111
|
+
oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
|
112
|
+
}
|
113
|
+
|
114
|
+
Properties props = new Properties();
|
115
|
+
props.putAll(oracleTask.getOptions());
|
116
|
+
|
117
|
+
props.setProperty("user", oracleTask.getUser());
|
118
|
+
logger.info("Connecting to {} options {}", url, props);
|
119
|
+
props.setProperty("password", oracleTask.getPassword());
|
120
|
+
|
121
|
+
return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
|
122
|
+
}
|
123
|
+
|
124
|
+
@Override
|
125
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
126
|
+
{
|
127
|
+
if (mergeKeys.isPresent()) {
|
128
|
+
throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
|
129
|
+
}
|
130
|
+
|
131
|
+
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
132
|
+
JdbcOutputConnector connector = getConnector(task, true);
|
133
|
+
|
134
|
+
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
135
|
+
OracleCharset charset;
|
136
|
+
try (JdbcOutputConnection connection = connector.connect(true)) {
|
137
|
+
charset = ((OracleOutputConnection)connection).getOracleCharset();
|
138
|
+
}
|
139
|
+
|
140
|
+
return new DirectBatchInsert(
|
141
|
+
String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
|
142
|
+
oracleTask.getUser(),
|
143
|
+
oracleTask.getPassword(),
|
144
|
+
oracleTask.getTable(),
|
145
|
+
charset,
|
146
|
+
oracleTask.getBatchSize());
|
147
|
+
}
|
148
|
+
|
149
|
+
return new StandardBatchInsert(getConnector(task, true), mergeKeys);
|
150
|
+
}
|
151
|
+
}
|
@@ -1,290 +1,290 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
import java.io.IOException;
|
4
|
-
import java.math.BigDecimal;
|
5
|
-
import java.sql.SQLException;
|
6
|
-
import java.sql.Types;
|
7
|
-
import java.text.DateFormat;
|
8
|
-
import java.text.SimpleDateFormat;
|
9
|
-
import java.util.ArrayList;
|
10
|
-
import java.util.Arrays;
|
11
|
-
import java.util.List;
|
12
|
-
import java.util.Calendar;
|
13
|
-
import org.embulk.output.jdbc.BatchInsert;
|
14
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
15
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
16
|
-
import org.embulk.output.oracle.oci.ColumnDefinition;
|
17
|
-
import org.embulk.output.oracle.oci.OCIManager;
|
18
|
-
import org.embulk.output.oracle.oci.OCIWrapper;
|
19
|
-
import org.embulk.output.oracle.oci.RowBuffer;
|
20
|
-
import org.embulk.output.oracle.oci.TableDefinition;
|
21
|
-
import org.embulk.spi.time.Timestamp;
|
22
|
-
import org.embulk.spi.Exec;
|
23
|
-
import org.slf4j.Logger;
|
24
|
-
|
25
|
-
public class DirectBatchInsert implements BatchInsert
|
26
|
-
{
|
27
|
-
private static OCIManager ociManager = new OCIManager();
|
28
|
-
|
29
|
-
private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
|
30
|
-
|
31
|
-
private final List<String> ociKey;
|
32
|
-
private final String database;
|
33
|
-
private final String user;
|
34
|
-
private final String password;
|
35
|
-
private final String table;
|
36
|
-
private final OracleCharset charset;
|
37
|
-
private final int batchSize;
|
38
|
-
private RowBuffer buffer;
|
39
|
-
private long totalRows;
|
40
|
-
private int rowSize;
|
41
|
-
private int batchWeight;
|
42
|
-
|
43
|
-
private DateFormat[] formats;
|
44
|
-
|
45
|
-
|
46
|
-
public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
|
47
|
-
{
|
48
|
-
this.database = database;
|
49
|
-
this.user = user;
|
50
|
-
this.password = password;
|
51
|
-
this.table = table;
|
52
|
-
this.charset = charset;
|
53
|
-
this.batchSize = batchSize;
|
54
|
-
|
55
|
-
ociKey = Arrays.asList(database, user, table);
|
56
|
-
}
|
57
|
-
|
58
|
-
@Override
|
59
|
-
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
60
|
-
{
|
61
|
-
|
62
|
-
/*
|
63
|
-
* available mappings
|
64
|
-
*
|
65
|
-
* boolean -> unused
|
66
|
-
* byte -> unused
|
67
|
-
* short -> unused
|
68
|
-
* int -> NUMBER
|
69
|
-
* long -> NUMBER
|
70
|
-
* BigDecimal -> NUMBER
|
71
|
-
* String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
|
72
|
-
* NString -> unused
|
73
|
-
* bytes -> unused
|
74
|
-
* SqlDate -> unused
|
75
|
-
* SqlTime -> unused
|
76
|
-
* SqlTimeStamp -> TIMESTAMP
|
77
|
-
*
|
78
|
-
*/
|
79
|
-
|
80
|
-
formats = new DateFormat[insertSchema.getCount()];
|
81
|
-
List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
|
82
|
-
java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
|
83
|
-
for (int i = 0; i < insertSchema.getCount(); i++) {
|
84
|
-
JdbcColumn insertColumn = insertSchema.getColumn(i);
|
85
|
-
switch (insertColumn.getSqlType()) {
|
86
|
-
case Types.CHAR:
|
87
|
-
case Types.VARCHAR:
|
88
|
-
case Types.LONGVARCHAR:
|
89
|
-
case Types.CLOB:
|
90
|
-
// TODO: CHAR(n CHAR)
|
91
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
92
|
-
ColumnDefinition.SQLT_CHR,
|
93
|
-
insertColumn.getSizeTypeParameter()));
|
94
|
-
break;
|
95
|
-
|
96
|
-
case Types.DECIMAL:
|
97
|
-
// sign + size
|
98
|
-
int size = 1 + insertColumn.getSizeTypeParameter();
|
99
|
-
if (insertColumn.getSizeTypeParameter() > 0) {
|
100
|
-
// decimal point
|
101
|
-
size += 1;
|
102
|
-
}
|
103
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
104
|
-
ColumnDefinition.SQLT_CHR,
|
105
|
-
size));
|
106
|
-
break;
|
107
|
-
|
108
|
-
case Types.DATE:
|
109
|
-
break;
|
110
|
-
|
111
|
-
case Types.TIMESTAMP:
|
112
|
-
String oracleFormat;
|
113
|
-
DateFormat javaFormat;
|
114
|
-
if (insertColumn.getSimpleTypeName().equals("DATE")) {
|
115
|
-
oracleFormat = "YYYY-MM-DD HH24:MI:SS";
|
116
|
-
javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
117
|
-
} else {
|
118
|
-
oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
|
119
|
-
javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
|
120
|
-
}
|
121
|
-
formats[i] = javaFormat;
|
122
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
123
|
-
ColumnDefinition.SQLT_CHR,
|
124
|
-
javaFormat.format(dummy).length(),
|
125
|
-
oracleFormat));
|
126
|
-
break;
|
127
|
-
|
128
|
-
default:
|
129
|
-
throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
|
130
|
-
}
|
131
|
-
|
132
|
-
}
|
133
|
-
|
134
|
-
rowSize = 0;
|
135
|
-
for (ColumnDefinition column : columns) {
|
136
|
-
rowSize += column.columnSize;
|
137
|
-
}
|
138
|
-
|
139
|
-
TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
|
140
|
-
ociManager.open(ociKey, database, user, password, tableDefinition);
|
141
|
-
|
142
|
-
buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
|
143
|
-
}
|
144
|
-
|
145
|
-
@Override
|
146
|
-
public int getBatchWeight()
|
147
|
-
{
|
148
|
-
return batchWeight;
|
149
|
-
}
|
150
|
-
|
151
|
-
@Override
|
152
|
-
public void add() throws IOException, SQLException
|
153
|
-
{
|
154
|
-
batchWeight += rowSize;
|
155
|
-
if (buffer.isFull()) {
|
156
|
-
flush();
|
157
|
-
}
|
158
|
-
}
|
159
|
-
|
160
|
-
@Override
|
161
|
-
public void close() throws IOException, SQLException
|
162
|
-
{
|
163
|
-
ociManager.close(ociKey);
|
164
|
-
}
|
165
|
-
|
166
|
-
@Override
|
167
|
-
public void flush() throws IOException, SQLException
|
168
|
-
{
|
169
|
-
if (buffer.getRowCount() > 0) {
|
170
|
-
try {
|
171
|
-
logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
|
172
|
-
|
173
|
-
long startTime = System.currentTimeMillis();
|
174
|
-
|
175
|
-
OCIWrapper oci = ociManager.get(ociKey);
|
176
|
-
synchronized (oci) {
|
177
|
-
oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
|
178
|
-
}
|
179
|
-
|
180
|
-
totalRows += buffer.getRowCount();
|
181
|
-
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
182
|
-
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
183
|
-
|
184
|
-
} finally {
|
185
|
-
buffer.clear();
|
186
|
-
batchWeight = 0;
|
187
|
-
}
|
188
|
-
}
|
189
|
-
}
|
190
|
-
|
191
|
-
@Override
|
192
|
-
public void finish() throws IOException, SQLException
|
193
|
-
{
|
194
|
-
flush();
|
195
|
-
}
|
196
|
-
|
197
|
-
@Override
|
198
|
-
public void setNull(int sqlType) throws IOException, SQLException
|
199
|
-
{
|
200
|
-
buffer.addValue("");
|
201
|
-
}
|
202
|
-
|
203
|
-
@Override
|
204
|
-
public void setBoolean(boolean v) throws IOException, SQLException
|
205
|
-
{
|
206
|
-
throw new SQLException("Unsupported");
|
207
|
-
}
|
208
|
-
|
209
|
-
@Override
|
210
|
-
public void setByte(byte v) throws IOException, SQLException
|
211
|
-
{
|
212
|
-
throw new SQLException("Unsupported");
|
213
|
-
}
|
214
|
-
|
215
|
-
@Override
|
216
|
-
public void setShort(short v) throws IOException, SQLException
|
217
|
-
{
|
218
|
-
throw new SQLException("Unsupported");
|
219
|
-
}
|
220
|
-
|
221
|
-
@Override
|
222
|
-
public void setInt(int v) throws IOException, SQLException
|
223
|
-
{
|
224
|
-
buffer.addValue(v);
|
225
|
-
}
|
226
|
-
|
227
|
-
@Override
|
228
|
-
public void setLong(long v) throws IOException, SQLException
|
229
|
-
{
|
230
|
-
buffer.addValue(Long.toString(v));
|
231
|
-
}
|
232
|
-
|
233
|
-
@Override
|
234
|
-
public void setFloat(float v) throws IOException, SQLException
|
235
|
-
{
|
236
|
-
throw new SQLException("Unsupported");
|
237
|
-
}
|
238
|
-
|
239
|
-
@Override
|
240
|
-
public void setDouble(double v) throws IOException, SQLException
|
241
|
-
{
|
242
|
-
throw new SQLException("Unsupported");
|
243
|
-
}
|
244
|
-
|
245
|
-
@Override
|
246
|
-
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
247
|
-
{
|
248
|
-
buffer.addValue(v);
|
249
|
-
}
|
250
|
-
|
251
|
-
@Override
|
252
|
-
public void setString(String v) throws IOException, SQLException
|
253
|
-
{
|
254
|
-
buffer.addValue(v);
|
255
|
-
}
|
256
|
-
|
257
|
-
@Override
|
258
|
-
public void setNString(String v) throws IOException, SQLException
|
259
|
-
{
|
260
|
-
throw new SQLException("Unsupported");
|
261
|
-
}
|
262
|
-
|
263
|
-
@Override
|
264
|
-
public void setBytes(byte[] v) throws IOException, SQLException
|
265
|
-
{
|
266
|
-
throw new SQLException("Unsupported");
|
267
|
-
}
|
268
|
-
|
269
|
-
@Override
|
270
|
-
public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
|
271
|
-
{
|
272
|
-
throw new SQLException("Unsupported");
|
273
|
-
}
|
274
|
-
|
275
|
-
@Override
|
276
|
-
public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
|
277
|
-
{
|
278
|
-
throw new SQLException("Unsupported");
|
279
|
-
}
|
280
|
-
|
281
|
-
@Override
|
282
|
-
public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
|
283
|
-
{
|
284
|
-
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
285
|
-
t.setNanos(v.getNano());
|
286
|
-
DateFormat format = formats[buffer.getCurrentColumn()];
|
287
|
-
format.setTimeZone(calendar.getTimeZone());
|
288
|
-
buffer.addValue(format.format(t));
|
289
|
-
}
|
290
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.math.BigDecimal;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import java.sql.Types;
|
7
|
+
import java.text.DateFormat;
|
8
|
+
import java.text.SimpleDateFormat;
|
9
|
+
import java.util.ArrayList;
|
10
|
+
import java.util.Arrays;
|
11
|
+
import java.util.List;
|
12
|
+
import java.util.Calendar;
|
13
|
+
import org.embulk.output.jdbc.BatchInsert;
|
14
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
15
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
16
|
+
import org.embulk.output.oracle.oci.ColumnDefinition;
|
17
|
+
import org.embulk.output.oracle.oci.OCIManager;
|
18
|
+
import org.embulk.output.oracle.oci.OCIWrapper;
|
19
|
+
import org.embulk.output.oracle.oci.RowBuffer;
|
20
|
+
import org.embulk.output.oracle.oci.TableDefinition;
|
21
|
+
import org.embulk.spi.time.Timestamp;
|
22
|
+
import org.embulk.spi.Exec;
|
23
|
+
import org.slf4j.Logger;
|
24
|
+
|
25
|
+
public class DirectBatchInsert implements BatchInsert
|
26
|
+
{
|
27
|
+
private static OCIManager ociManager = new OCIManager();
|
28
|
+
|
29
|
+
private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
|
30
|
+
|
31
|
+
private final List<String> ociKey;
|
32
|
+
private final String database;
|
33
|
+
private final String user;
|
34
|
+
private final String password;
|
35
|
+
private final String table;
|
36
|
+
private final OracleCharset charset;
|
37
|
+
private final int batchSize;
|
38
|
+
private RowBuffer buffer;
|
39
|
+
private long totalRows;
|
40
|
+
private int rowSize;
|
41
|
+
private int batchWeight;
|
42
|
+
|
43
|
+
private DateFormat[] formats;
|
44
|
+
|
45
|
+
|
46
|
+
public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
|
47
|
+
{
|
48
|
+
this.database = database;
|
49
|
+
this.user = user;
|
50
|
+
this.password = password;
|
51
|
+
this.table = table;
|
52
|
+
this.charset = charset;
|
53
|
+
this.batchSize = batchSize;
|
54
|
+
|
55
|
+
ociKey = Arrays.asList(database, user, table);
|
56
|
+
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
60
|
+
{
|
61
|
+
|
62
|
+
/*
|
63
|
+
* available mappings
|
64
|
+
*
|
65
|
+
* boolean -> unused
|
66
|
+
* byte -> unused
|
67
|
+
* short -> unused
|
68
|
+
* int -> NUMBER
|
69
|
+
* long -> NUMBER
|
70
|
+
* BigDecimal -> NUMBER
|
71
|
+
* String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
|
72
|
+
* NString -> unused
|
73
|
+
* bytes -> unused
|
74
|
+
* SqlDate -> unused
|
75
|
+
* SqlTime -> unused
|
76
|
+
* SqlTimeStamp -> TIMESTAMP
|
77
|
+
*
|
78
|
+
*/
|
79
|
+
|
80
|
+
formats = new DateFormat[insertSchema.getCount()];
|
81
|
+
List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
|
82
|
+
java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
|
83
|
+
for (int i = 0; i < insertSchema.getCount(); i++) {
|
84
|
+
JdbcColumn insertColumn = insertSchema.getColumn(i);
|
85
|
+
switch (insertColumn.getSqlType()) {
|
86
|
+
case Types.CHAR:
|
87
|
+
case Types.VARCHAR:
|
88
|
+
case Types.LONGVARCHAR:
|
89
|
+
case Types.CLOB:
|
90
|
+
// TODO: CHAR(n CHAR)
|
91
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
92
|
+
ColumnDefinition.SQLT_CHR,
|
93
|
+
insertColumn.getSizeTypeParameter()));
|
94
|
+
break;
|
95
|
+
|
96
|
+
case Types.DECIMAL:
|
97
|
+
// sign + size
|
98
|
+
int size = 1 + insertColumn.getSizeTypeParameter();
|
99
|
+
if (insertColumn.getSizeTypeParameter() > 0) {
|
100
|
+
// decimal point
|
101
|
+
size += 1;
|
102
|
+
}
|
103
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
104
|
+
ColumnDefinition.SQLT_CHR,
|
105
|
+
size));
|
106
|
+
break;
|
107
|
+
|
108
|
+
case Types.DATE:
|
109
|
+
break;
|
110
|
+
|
111
|
+
case Types.TIMESTAMP:
|
112
|
+
String oracleFormat;
|
113
|
+
DateFormat javaFormat;
|
114
|
+
if (insertColumn.getSimpleTypeName().equals("DATE")) {
|
115
|
+
oracleFormat = "YYYY-MM-DD HH24:MI:SS";
|
116
|
+
javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
117
|
+
} else {
|
118
|
+
oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
|
119
|
+
javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
|
120
|
+
}
|
121
|
+
formats[i] = javaFormat;
|
122
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
123
|
+
ColumnDefinition.SQLT_CHR,
|
124
|
+
javaFormat.format(dummy).length(),
|
125
|
+
oracleFormat));
|
126
|
+
break;
|
127
|
+
|
128
|
+
default:
|
129
|
+
throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
|
130
|
+
}
|
131
|
+
|
132
|
+
}
|
133
|
+
|
134
|
+
rowSize = 0;
|
135
|
+
for (ColumnDefinition column : columns) {
|
136
|
+
rowSize += column.columnSize;
|
137
|
+
}
|
138
|
+
|
139
|
+
TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
|
140
|
+
ociManager.open(ociKey, database, user, password, tableDefinition);
|
141
|
+
|
142
|
+
buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
|
143
|
+
}
|
144
|
+
|
145
|
+
@Override
|
146
|
+
public int getBatchWeight()
|
147
|
+
{
|
148
|
+
return batchWeight;
|
149
|
+
}
|
150
|
+
|
151
|
+
@Override
|
152
|
+
public void add() throws IOException, SQLException
|
153
|
+
{
|
154
|
+
batchWeight += rowSize;
|
155
|
+
if (buffer.isFull()) {
|
156
|
+
flush();
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
@Override
|
161
|
+
public void close() throws IOException, SQLException
|
162
|
+
{
|
163
|
+
ociManager.close(ociKey);
|
164
|
+
}
|
165
|
+
|
166
|
+
@Override
|
167
|
+
public void flush() throws IOException, SQLException
|
168
|
+
{
|
169
|
+
if (buffer.getRowCount() > 0) {
|
170
|
+
try {
|
171
|
+
logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
|
172
|
+
|
173
|
+
long startTime = System.currentTimeMillis();
|
174
|
+
|
175
|
+
OCIWrapper oci = ociManager.get(ociKey);
|
176
|
+
synchronized (oci) {
|
177
|
+
oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
|
178
|
+
}
|
179
|
+
|
180
|
+
totalRows += buffer.getRowCount();
|
181
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
182
|
+
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
183
|
+
|
184
|
+
} finally {
|
185
|
+
buffer.clear();
|
186
|
+
batchWeight = 0;
|
187
|
+
}
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
@Override
|
192
|
+
public void finish() throws IOException, SQLException
|
193
|
+
{
|
194
|
+
flush();
|
195
|
+
}
|
196
|
+
|
197
|
+
@Override
|
198
|
+
public void setNull(int sqlType) throws IOException, SQLException
|
199
|
+
{
|
200
|
+
buffer.addValue("");
|
201
|
+
}
|
202
|
+
|
203
|
+
@Override
|
204
|
+
public void setBoolean(boolean v) throws IOException, SQLException
|
205
|
+
{
|
206
|
+
throw new SQLException("Unsupported");
|
207
|
+
}
|
208
|
+
|
209
|
+
@Override
|
210
|
+
public void setByte(byte v) throws IOException, SQLException
|
211
|
+
{
|
212
|
+
throw new SQLException("Unsupported");
|
213
|
+
}
|
214
|
+
|
215
|
+
@Override
|
216
|
+
public void setShort(short v) throws IOException, SQLException
|
217
|
+
{
|
218
|
+
throw new SQLException("Unsupported");
|
219
|
+
}
|
220
|
+
|
221
|
+
@Override
|
222
|
+
public void setInt(int v) throws IOException, SQLException
|
223
|
+
{
|
224
|
+
buffer.addValue(v);
|
225
|
+
}
|
226
|
+
|
227
|
+
@Override
|
228
|
+
public void setLong(long v) throws IOException, SQLException
|
229
|
+
{
|
230
|
+
buffer.addValue(Long.toString(v));
|
231
|
+
}
|
232
|
+
|
233
|
+
@Override
|
234
|
+
public void setFloat(float v) throws IOException, SQLException
|
235
|
+
{
|
236
|
+
throw new SQLException("Unsupported");
|
237
|
+
}
|
238
|
+
|
239
|
+
@Override
|
240
|
+
public void setDouble(double v) throws IOException, SQLException
|
241
|
+
{
|
242
|
+
throw new SQLException("Unsupported");
|
243
|
+
}
|
244
|
+
|
245
|
+
@Override
|
246
|
+
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
247
|
+
{
|
248
|
+
buffer.addValue(v);
|
249
|
+
}
|
250
|
+
|
251
|
+
@Override
|
252
|
+
public void setString(String v) throws IOException, SQLException
|
253
|
+
{
|
254
|
+
buffer.addValue(v);
|
255
|
+
}
|
256
|
+
|
257
|
+
@Override
|
258
|
+
public void setNString(String v) throws IOException, SQLException
|
259
|
+
{
|
260
|
+
throw new SQLException("Unsupported");
|
261
|
+
}
|
262
|
+
|
263
|
+
@Override
|
264
|
+
public void setBytes(byte[] v) throws IOException, SQLException
|
265
|
+
{
|
266
|
+
throw new SQLException("Unsupported");
|
267
|
+
}
|
268
|
+
|
269
|
+
@Override
|
270
|
+
public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
|
271
|
+
{
|
272
|
+
throw new SQLException("Unsupported");
|
273
|
+
}
|
274
|
+
|
275
|
+
@Override
|
276
|
+
public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
|
277
|
+
{
|
278
|
+
throw new SQLException("Unsupported");
|
279
|
+
}
|
280
|
+
|
281
|
+
@Override
|
282
|
+
public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
|
283
|
+
{
|
284
|
+
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
285
|
+
t.setNanos(v.getNano());
|
286
|
+
DateFormat format = formats[buffer.getCurrentColumn()];
|
287
|
+
format.setTimeZone(calendar.getTimeZone());
|
288
|
+
buffer.addValue(format.format(t));
|
289
|
+
}
|
290
|
+
}
|