embulk-output-oracle 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +160 -160
- data/build.gradle +6 -6
- data/classpath/{embulk-output-jdbc-0.4.0.jar → embulk-output-jdbc-0.4.1.jar} +0 -0
- data/classpath/{embulk-output-oracle-0.4.0.jar → embulk-output-oracle-0.4.1.jar} +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +153 -153
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -165
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +124 -120
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +105 -100
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +38 -14
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +540 -484
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -0
- data/src/test/resources/yml/test-insert-direct-empty.yml +25 -0
- data/src/test/resources/yml/{test-insert-oci-split.yml → test-insert-direct-oci-method-split.yml} +28 -28
- data/src/test/resources/yml/{test-insert-oci.yml → test-insert-direct-oci-method.yml} +28 -28
- data/src/test/resources/yml/test-insert-direct.yml +25 -26
- data/src/test/resources/yml/test-insert-empty.yml +27 -0
- data/src/test/resources/yml/test-insert.yml +27 -25
- data/src/test/resources/yml/test-replace-empty.yml +31 -0
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name.yml +31 -31
- data/src/test/resources/yml/test-replace.yml +31 -31
- data/src/test/resources/yml/test-string-timestamp.yml +28 -28
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +10 -6
@@ -1,153 +1,153 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import java.util.List;
|
4
|
-
import java.util.Properties;
|
5
|
-
import java.io.IOException;
|
6
|
-
import java.nio.ByteBuffer;
|
7
|
-
import java.sql.SQLException;
|
8
|
-
import com.google.common.base.Optional;
|
9
|
-
import com.google.common.collect.ImmutableSet;
|
10
|
-
import org.embulk.config.Config;
|
11
|
-
import org.embulk.config.ConfigDefault;
|
12
|
-
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
13
|
-
import org.embulk.output.jdbc.BatchInsert;
|
14
|
-
import org.embulk.output.jdbc.JdbcOutputConnection;
|
15
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
16
|
-
import org.embulk.output.jdbc.StandardBatchInsert;
|
17
|
-
import org.embulk.output.oracle.DirectBatchInsert;
|
18
|
-
import org.embulk.output.oracle.InsertMethod;
|
19
|
-
import org.embulk.output.oracle.OracleCharset;
|
20
|
-
import org.embulk.output.oracle.OracleOutputConnection;
|
21
|
-
import org.embulk.output.oracle.OracleOutputConnector;
|
22
|
-
import org.embulk.spi.PageReader;
|
23
|
-
|
24
|
-
public class OracleOutputPlugin
|
25
|
-
extends AbstractJdbcOutputPlugin
|
26
|
-
{
|
27
|
-
public interface OraclePluginTask
|
28
|
-
extends PluginTask
|
29
|
-
{
|
30
|
-
@Config("driver_path")
|
31
|
-
@ConfigDefault("null")
|
32
|
-
public Optional<String> getDriverPath();
|
33
|
-
|
34
|
-
@Config("host")
|
35
|
-
@ConfigDefault("null")
|
36
|
-
public Optional<String> getHost();
|
37
|
-
|
38
|
-
@Config("port")
|
39
|
-
@ConfigDefault("1521")
|
40
|
-
public int getPort();
|
41
|
-
|
42
|
-
@Config("database")
|
43
|
-
@ConfigDefault("null")
|
44
|
-
public Optional<String> getDatabase();
|
45
|
-
|
46
|
-
@Config("url")
|
47
|
-
@ConfigDefault("null")
|
48
|
-
public Optional<String> getUrl();
|
49
|
-
|
50
|
-
@Config("user")
|
51
|
-
public String getUser();
|
52
|
-
|
53
|
-
@Config("password")
|
54
|
-
@ConfigDefault("\"\"")
|
55
|
-
public String getPassword();
|
56
|
-
|
57
|
-
@Config("insert_method")
|
58
|
-
@ConfigDefault("\"normal\"")
|
59
|
-
public InsertMethod getInsertMethod();
|
60
|
-
}
|
61
|
-
|
62
|
-
@Override
|
63
|
-
protected Class<? extends PluginTask> getTaskClass()
|
64
|
-
{
|
65
|
-
return OraclePluginTask.class;
|
66
|
-
}
|
67
|
-
|
68
|
-
@Override
|
69
|
-
protected Features getFeatures(PluginTask task)
|
70
|
-
{
|
71
|
-
return new Features()
|
72
|
-
.setMaxTableNameLength(30)
|
73
|
-
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
74
|
-
.setIgnoreMergeKeys(false);
|
75
|
-
}
|
76
|
-
|
77
|
-
@Override
|
78
|
-
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
79
|
-
{
|
80
|
-
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
81
|
-
|
82
|
-
if (oracleTask.getDriverPath().isPresent()) {
|
83
|
-
loadDriverJar(oracleTask.getDriverPath().get());
|
84
|
-
}
|
85
|
-
|
86
|
-
String url;
|
87
|
-
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
88
|
-
if (!oracleTask.getHost().isPresent()) {
|
89
|
-
throw new IllegalArgumentException("Field 'host' is not set.");
|
90
|
-
}
|
91
|
-
if (!oracleTask.getDatabase().isPresent()) {
|
92
|
-
throw new IllegalArgumentException("Field 'database' is not set.");
|
93
|
-
}
|
94
|
-
} else {
|
95
|
-
if (oracleTask.getUrl().isPresent()) {
|
96
|
-
if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
|
97
|
-
throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
|
98
|
-
}
|
99
|
-
} else {
|
100
|
-
if (!oracleTask.getHost().isPresent()) {
|
101
|
-
throw new IllegalArgumentException("Field 'host' is not set.");
|
102
|
-
}
|
103
|
-
if (!oracleTask.getDatabase().isPresent()) {
|
104
|
-
throw new IllegalArgumentException("Field 'database' is not set.");
|
105
|
-
}
|
106
|
-
}
|
107
|
-
}
|
108
|
-
|
109
|
-
if (oracleTask.getUrl().isPresent()) {
|
110
|
-
url = oracleTask.getUrl().get();
|
111
|
-
} else {
|
112
|
-
url = String.format("jdbc:oracle:thin:@%s:%d:%s",
|
113
|
-
oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
|
114
|
-
}
|
115
|
-
|
116
|
-
Properties props = new Properties();
|
117
|
-
props.putAll(oracleTask.getOptions());
|
118
|
-
|
119
|
-
props.setProperty("user", oracleTask.getUser());
|
120
|
-
logger.info("Connecting to {} options {}", url, props);
|
121
|
-
props.setProperty("password", oracleTask.getPassword());
|
122
|
-
|
123
|
-
return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
|
124
|
-
}
|
125
|
-
|
126
|
-
@Override
|
127
|
-
protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
128
|
-
{
|
129
|
-
if (mergeKeys.isPresent()) {
|
130
|
-
throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
|
131
|
-
}
|
132
|
-
|
133
|
-
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
134
|
-
JdbcOutputConnector connector = getConnector(task, true);
|
135
|
-
|
136
|
-
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
137
|
-
OracleCharset charset;
|
138
|
-
try (JdbcOutputConnection connection = connector.connect(true)) {
|
139
|
-
charset = ((OracleOutputConnection)connection).getOracleCharset();
|
140
|
-
}
|
141
|
-
|
142
|
-
return new DirectBatchInsert(
|
143
|
-
String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
|
144
|
-
oracleTask.getUser(),
|
145
|
-
oracleTask.getPassword(),
|
146
|
-
oracleTask.getTable(),
|
147
|
-
charset,
|
148
|
-
oracleTask.getBatchSize());
|
149
|
-
}
|
150
|
-
|
151
|
-
return new StandardBatchInsert(getConnector(task, true), mergeKeys);
|
152
|
-
}
|
153
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Properties;
|
5
|
+
import java.io.IOException;
|
6
|
+
import java.nio.ByteBuffer;
|
7
|
+
import java.sql.SQLException;
|
8
|
+
import com.google.common.base.Optional;
|
9
|
+
import com.google.common.collect.ImmutableSet;
|
10
|
+
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
12
|
+
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
13
|
+
import org.embulk.output.jdbc.BatchInsert;
|
14
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
15
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
16
|
+
import org.embulk.output.jdbc.StandardBatchInsert;
|
17
|
+
import org.embulk.output.oracle.DirectBatchInsert;
|
18
|
+
import org.embulk.output.oracle.InsertMethod;
|
19
|
+
import org.embulk.output.oracle.OracleCharset;
|
20
|
+
import org.embulk.output.oracle.OracleOutputConnection;
|
21
|
+
import org.embulk.output.oracle.OracleOutputConnector;
|
22
|
+
import org.embulk.spi.PageReader;
|
23
|
+
|
24
|
+
public class OracleOutputPlugin
|
25
|
+
extends AbstractJdbcOutputPlugin
|
26
|
+
{
|
27
|
+
public interface OraclePluginTask
|
28
|
+
extends PluginTask
|
29
|
+
{
|
30
|
+
@Config("driver_path")
|
31
|
+
@ConfigDefault("null")
|
32
|
+
public Optional<String> getDriverPath();
|
33
|
+
|
34
|
+
@Config("host")
|
35
|
+
@ConfigDefault("null")
|
36
|
+
public Optional<String> getHost();
|
37
|
+
|
38
|
+
@Config("port")
|
39
|
+
@ConfigDefault("1521")
|
40
|
+
public int getPort();
|
41
|
+
|
42
|
+
@Config("database")
|
43
|
+
@ConfigDefault("null")
|
44
|
+
public Optional<String> getDatabase();
|
45
|
+
|
46
|
+
@Config("url")
|
47
|
+
@ConfigDefault("null")
|
48
|
+
public Optional<String> getUrl();
|
49
|
+
|
50
|
+
@Config("user")
|
51
|
+
public String getUser();
|
52
|
+
|
53
|
+
@Config("password")
|
54
|
+
@ConfigDefault("\"\"")
|
55
|
+
public String getPassword();
|
56
|
+
|
57
|
+
@Config("insert_method")
|
58
|
+
@ConfigDefault("\"normal\"")
|
59
|
+
public InsertMethod getInsertMethod();
|
60
|
+
}
|
61
|
+
|
62
|
+
@Override
|
63
|
+
protected Class<? extends PluginTask> getTaskClass()
|
64
|
+
{
|
65
|
+
return OraclePluginTask.class;
|
66
|
+
}
|
67
|
+
|
68
|
+
@Override
|
69
|
+
protected Features getFeatures(PluginTask task)
|
70
|
+
{
|
71
|
+
return new Features()
|
72
|
+
.setMaxTableNameLength(30)
|
73
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
74
|
+
.setIgnoreMergeKeys(false);
|
75
|
+
}
|
76
|
+
|
77
|
+
@Override
|
78
|
+
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
79
|
+
{
|
80
|
+
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
81
|
+
|
82
|
+
if (oracleTask.getDriverPath().isPresent()) {
|
83
|
+
loadDriverJar(oracleTask.getDriverPath().get());
|
84
|
+
}
|
85
|
+
|
86
|
+
String url;
|
87
|
+
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
88
|
+
if (!oracleTask.getHost().isPresent()) {
|
89
|
+
throw new IllegalArgumentException("Field 'host' is not set.");
|
90
|
+
}
|
91
|
+
if (!oracleTask.getDatabase().isPresent()) {
|
92
|
+
throw new IllegalArgumentException("Field 'database' is not set.");
|
93
|
+
}
|
94
|
+
} else {
|
95
|
+
if (oracleTask.getUrl().isPresent()) {
|
96
|
+
if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
|
97
|
+
throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
|
98
|
+
}
|
99
|
+
} else {
|
100
|
+
if (!oracleTask.getHost().isPresent()) {
|
101
|
+
throw new IllegalArgumentException("Field 'host' is not set.");
|
102
|
+
}
|
103
|
+
if (!oracleTask.getDatabase().isPresent()) {
|
104
|
+
throw new IllegalArgumentException("Field 'database' is not set.");
|
105
|
+
}
|
106
|
+
}
|
107
|
+
}
|
108
|
+
|
109
|
+
if (oracleTask.getUrl().isPresent()) {
|
110
|
+
url = oracleTask.getUrl().get();
|
111
|
+
} else {
|
112
|
+
url = String.format("jdbc:oracle:thin:@%s:%d:%s",
|
113
|
+
oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
|
114
|
+
}
|
115
|
+
|
116
|
+
Properties props = new Properties();
|
117
|
+
props.putAll(oracleTask.getOptions());
|
118
|
+
|
119
|
+
props.setProperty("user", oracleTask.getUser());
|
120
|
+
logger.info("Connecting to {} options {}", url, props);
|
121
|
+
props.setProperty("password", oracleTask.getPassword());
|
122
|
+
|
123
|
+
return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
|
124
|
+
}
|
125
|
+
|
126
|
+
@Override
|
127
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
128
|
+
{
|
129
|
+
if (mergeKeys.isPresent()) {
|
130
|
+
throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
|
131
|
+
}
|
132
|
+
|
133
|
+
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
134
|
+
JdbcOutputConnector connector = getConnector(task, true);
|
135
|
+
|
136
|
+
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
137
|
+
OracleCharset charset;
|
138
|
+
try (JdbcOutputConnection connection = connector.connect(true)) {
|
139
|
+
charset = ((OracleOutputConnection)connection).getOracleCharset();
|
140
|
+
}
|
141
|
+
|
142
|
+
return new DirectBatchInsert(
|
143
|
+
String.format("%s:%d/%s", oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get()),
|
144
|
+
oracleTask.getUser(),
|
145
|
+
oracleTask.getPassword(),
|
146
|
+
oracleTask.getTable(),
|
147
|
+
charset,
|
148
|
+
oracleTask.getBatchSize());
|
149
|
+
}
|
150
|
+
|
151
|
+
return new StandardBatchInsert(getConnector(task, true), mergeKeys);
|
152
|
+
}
|
153
|
+
}
|
@@ -1,290 +1,290 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
import java.io.IOException;
|
4
|
-
import java.math.BigDecimal;
|
5
|
-
import java.sql.SQLException;
|
6
|
-
import java.sql.Types;
|
7
|
-
import java.text.DateFormat;
|
8
|
-
import java.text.SimpleDateFormat;
|
9
|
-
import java.util.ArrayList;
|
10
|
-
import java.util.Arrays;
|
11
|
-
import java.util.List;
|
12
|
-
import java.util.Calendar;
|
13
|
-
import org.embulk.output.jdbc.BatchInsert;
|
14
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
15
|
-
import org.embulk.output.jdbc.JdbcSchema;
|
16
|
-
import org.embulk.output.oracle.oci.ColumnDefinition;
|
17
|
-
import org.embulk.output.oracle.oci.OCIManager;
|
18
|
-
import org.embulk.output.oracle.oci.OCIWrapper;
|
19
|
-
import org.embulk.output.oracle.oci.RowBuffer;
|
20
|
-
import org.embulk.output.oracle.oci.TableDefinition;
|
21
|
-
import org.embulk.spi.time.Timestamp;
|
22
|
-
import org.embulk.spi.Exec;
|
23
|
-
import org.slf4j.Logger;
|
24
|
-
|
25
|
-
public class DirectBatchInsert implements BatchInsert
|
26
|
-
{
|
27
|
-
private static OCIManager ociManager = new OCIManager();
|
28
|
-
|
29
|
-
private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
|
30
|
-
|
31
|
-
private final List<String> ociKey;
|
32
|
-
private final String database;
|
33
|
-
private final String user;
|
34
|
-
private final String password;
|
35
|
-
private final String table;
|
36
|
-
private final OracleCharset charset;
|
37
|
-
private final int batchSize;
|
38
|
-
private RowBuffer buffer;
|
39
|
-
private long totalRows;
|
40
|
-
private int rowSize;
|
41
|
-
private int batchWeight;
|
42
|
-
|
43
|
-
private DateFormat[] formats;
|
44
|
-
|
45
|
-
|
46
|
-
public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
|
47
|
-
{
|
48
|
-
this.database = database;
|
49
|
-
this.user = user;
|
50
|
-
this.password = password;
|
51
|
-
this.table = table;
|
52
|
-
this.charset = charset;
|
53
|
-
this.batchSize = batchSize;
|
54
|
-
|
55
|
-
ociKey = Arrays.asList(database, user, table);
|
56
|
-
}
|
57
|
-
|
58
|
-
@Override
|
59
|
-
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
60
|
-
{
|
61
|
-
|
62
|
-
/*
|
63
|
-
* available mappings
|
64
|
-
*
|
65
|
-
* boolean -> unused
|
66
|
-
* byte -> unused
|
67
|
-
* short -> unused
|
68
|
-
* int -> NUMBER
|
69
|
-
* long -> NUMBER
|
70
|
-
* BigDecimal -> NUMBER
|
71
|
-
* String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
|
72
|
-
* NString -> unused
|
73
|
-
* bytes -> unused
|
74
|
-
* SqlDate -> unused
|
75
|
-
* SqlTime -> unused
|
76
|
-
* SqlTimeStamp -> TIMESTAMP
|
77
|
-
*
|
78
|
-
*/
|
79
|
-
|
80
|
-
formats = new DateFormat[insertSchema.getCount()];
|
81
|
-
List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
|
82
|
-
java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
|
83
|
-
for (int i = 0; i < insertSchema.getCount(); i++) {
|
84
|
-
JdbcColumn insertColumn = insertSchema.getColumn(i);
|
85
|
-
switch (insertColumn.getSqlType()) {
|
86
|
-
case Types.CHAR:
|
87
|
-
case Types.VARCHAR:
|
88
|
-
case Types.LONGVARCHAR:
|
89
|
-
case Types.CLOB:
|
90
|
-
// TODO: CHAR(n CHAR)
|
91
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
92
|
-
ColumnDefinition.SQLT_CHR,
|
93
|
-
insertColumn.getSizeTypeParameter()));
|
94
|
-
break;
|
95
|
-
|
96
|
-
case Types.DECIMAL:
|
97
|
-
// sign + size
|
98
|
-
int size = 1 + insertColumn.getSizeTypeParameter();
|
99
|
-
if (insertColumn.getSizeTypeParameter() > 0) {
|
100
|
-
// decimal point
|
101
|
-
size += 1;
|
102
|
-
}
|
103
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
104
|
-
ColumnDefinition.SQLT_CHR,
|
105
|
-
size));
|
106
|
-
break;
|
107
|
-
|
108
|
-
case Types.DATE:
|
109
|
-
break;
|
110
|
-
|
111
|
-
case Types.TIMESTAMP:
|
112
|
-
String oracleFormat;
|
113
|
-
DateFormat javaFormat;
|
114
|
-
if (insertColumn.getSimpleTypeName().equals("DATE")) {
|
115
|
-
oracleFormat = "YYYY-MM-DD HH24:MI:SS";
|
116
|
-
javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
117
|
-
} else {
|
118
|
-
oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
|
119
|
-
javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
|
120
|
-
}
|
121
|
-
formats[i] = javaFormat;
|
122
|
-
columns.add(new ColumnDefinition(insertColumn.getName(),
|
123
|
-
ColumnDefinition.SQLT_CHR,
|
124
|
-
javaFormat.format(dummy).length(),
|
125
|
-
oracleFormat));
|
126
|
-
break;
|
127
|
-
|
128
|
-
default:
|
129
|
-
throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
|
130
|
-
}
|
131
|
-
|
132
|
-
}
|
133
|
-
|
134
|
-
rowSize = 0;
|
135
|
-
for (ColumnDefinition column : columns) {
|
136
|
-
rowSize += column.columnSize;
|
137
|
-
}
|
138
|
-
|
139
|
-
TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
|
140
|
-
ociManager.open(ociKey, database, user, password, tableDefinition);
|
141
|
-
|
142
|
-
buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
|
143
|
-
}
|
144
|
-
|
145
|
-
@Override
|
146
|
-
public int getBatchWeight()
|
147
|
-
{
|
148
|
-
return batchWeight;
|
149
|
-
}
|
150
|
-
|
151
|
-
@Override
|
152
|
-
public void add() throws IOException, SQLException
|
153
|
-
{
|
154
|
-
batchWeight += rowSize;
|
155
|
-
if (buffer.isFull()) {
|
156
|
-
flush();
|
157
|
-
}
|
158
|
-
}
|
159
|
-
|
160
|
-
@Override
|
161
|
-
public void close() throws IOException, SQLException
|
162
|
-
{
|
163
|
-
ociManager.close(ociKey);
|
164
|
-
}
|
165
|
-
|
166
|
-
@Override
|
167
|
-
public void flush() throws IOException, SQLException
|
168
|
-
{
|
169
|
-
if (buffer.getRowCount() > 0) {
|
170
|
-
try {
|
171
|
-
logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
|
172
|
-
|
173
|
-
long startTime = System.currentTimeMillis();
|
174
|
-
|
175
|
-
OCIWrapper oci = ociManager.get(ociKey);
|
176
|
-
synchronized (oci) {
|
177
|
-
oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
|
178
|
-
}
|
179
|
-
|
180
|
-
totalRows += buffer.getRowCount();
|
181
|
-
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
182
|
-
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
183
|
-
|
184
|
-
} finally {
|
185
|
-
buffer.clear();
|
186
|
-
batchWeight = 0;
|
187
|
-
}
|
188
|
-
}
|
189
|
-
}
|
190
|
-
|
191
|
-
@Override
|
192
|
-
public void finish() throws IOException, SQLException
|
193
|
-
{
|
194
|
-
flush();
|
195
|
-
}
|
196
|
-
|
197
|
-
@Override
|
198
|
-
public void setNull(int sqlType) throws IOException, SQLException
|
199
|
-
{
|
200
|
-
buffer.addValue("");
|
201
|
-
}
|
202
|
-
|
203
|
-
@Override
|
204
|
-
public void setBoolean(boolean v) throws IOException, SQLException
|
205
|
-
{
|
206
|
-
throw new SQLException("Unsupported");
|
207
|
-
}
|
208
|
-
|
209
|
-
@Override
|
210
|
-
public void setByte(byte v) throws IOException, SQLException
|
211
|
-
{
|
212
|
-
throw new SQLException("Unsupported");
|
213
|
-
}
|
214
|
-
|
215
|
-
@Override
|
216
|
-
public void setShort(short v) throws IOException, SQLException
|
217
|
-
{
|
218
|
-
throw new SQLException("Unsupported");
|
219
|
-
}
|
220
|
-
|
221
|
-
@Override
|
222
|
-
public void setInt(int v) throws IOException, SQLException
|
223
|
-
{
|
224
|
-
buffer.addValue(v);
|
225
|
-
}
|
226
|
-
|
227
|
-
@Override
|
228
|
-
public void setLong(long v) throws IOException, SQLException
|
229
|
-
{
|
230
|
-
buffer.addValue(Long.toString(v));
|
231
|
-
}
|
232
|
-
|
233
|
-
@Override
|
234
|
-
public void setFloat(float v) throws IOException, SQLException
|
235
|
-
{
|
236
|
-
throw new SQLException("Unsupported");
|
237
|
-
}
|
238
|
-
|
239
|
-
@Override
|
240
|
-
public void setDouble(double v) throws IOException, SQLException
|
241
|
-
{
|
242
|
-
throw new SQLException("Unsupported");
|
243
|
-
}
|
244
|
-
|
245
|
-
@Override
|
246
|
-
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
247
|
-
{
|
248
|
-
buffer.addValue(v);
|
249
|
-
}
|
250
|
-
|
251
|
-
@Override
|
252
|
-
public void setString(String v) throws IOException, SQLException
|
253
|
-
{
|
254
|
-
buffer.addValue(v);
|
255
|
-
}
|
256
|
-
|
257
|
-
@Override
|
258
|
-
public void setNString(String v) throws IOException, SQLException
|
259
|
-
{
|
260
|
-
throw new SQLException("Unsupported");
|
261
|
-
}
|
262
|
-
|
263
|
-
@Override
|
264
|
-
public void setBytes(byte[] v) throws IOException, SQLException
|
265
|
-
{
|
266
|
-
throw new SQLException("Unsupported");
|
267
|
-
}
|
268
|
-
|
269
|
-
@Override
|
270
|
-
public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
|
271
|
-
{
|
272
|
-
throw new SQLException("Unsupported");
|
273
|
-
}
|
274
|
-
|
275
|
-
@Override
|
276
|
-
public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
|
277
|
-
{
|
278
|
-
throw new SQLException("Unsupported");
|
279
|
-
}
|
280
|
-
|
281
|
-
@Override
|
282
|
-
public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
|
283
|
-
{
|
284
|
-
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
285
|
-
t.setNanos(v.getNano());
|
286
|
-
DateFormat format = formats[buffer.getCurrentColumn()];
|
287
|
-
format.setTimeZone(calendar.getTimeZone());
|
288
|
-
buffer.addValue(format.format(t));
|
289
|
-
}
|
290
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.math.BigDecimal;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import java.sql.Types;
|
7
|
+
import java.text.DateFormat;
|
8
|
+
import java.text.SimpleDateFormat;
|
9
|
+
import java.util.ArrayList;
|
10
|
+
import java.util.Arrays;
|
11
|
+
import java.util.List;
|
12
|
+
import java.util.Calendar;
|
13
|
+
import org.embulk.output.jdbc.BatchInsert;
|
14
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
15
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
16
|
+
import org.embulk.output.oracle.oci.ColumnDefinition;
|
17
|
+
import org.embulk.output.oracle.oci.OCIManager;
|
18
|
+
import org.embulk.output.oracle.oci.OCIWrapper;
|
19
|
+
import org.embulk.output.oracle.oci.RowBuffer;
|
20
|
+
import org.embulk.output.oracle.oci.TableDefinition;
|
21
|
+
import org.embulk.spi.time.Timestamp;
|
22
|
+
import org.embulk.spi.Exec;
|
23
|
+
import org.slf4j.Logger;
|
24
|
+
|
25
|
+
public class DirectBatchInsert implements BatchInsert
|
26
|
+
{
|
27
|
+
private static OCIManager ociManager = new OCIManager();
|
28
|
+
|
29
|
+
private final Logger logger = Exec.getLogger(DirectBatchInsert.class);
|
30
|
+
|
31
|
+
private final List<String> ociKey;
|
32
|
+
private final String database;
|
33
|
+
private final String user;
|
34
|
+
private final String password;
|
35
|
+
private final String table;
|
36
|
+
private final OracleCharset charset;
|
37
|
+
private final int batchSize;
|
38
|
+
private RowBuffer buffer;
|
39
|
+
private long totalRows;
|
40
|
+
private int rowSize;
|
41
|
+
private int batchWeight;
|
42
|
+
|
43
|
+
private DateFormat[] formats;
|
44
|
+
|
45
|
+
|
46
|
+
public DirectBatchInsert(String database, String user, String password, String table, OracleCharset charset, int batchSize)
|
47
|
+
{
|
48
|
+
this.database = database;
|
49
|
+
this.user = user;
|
50
|
+
this.password = password;
|
51
|
+
this.table = table;
|
52
|
+
this.charset = charset;
|
53
|
+
this.batchSize = batchSize;
|
54
|
+
|
55
|
+
ociKey = Arrays.asList(database, user, table);
|
56
|
+
}
|
57
|
+
|
58
|
+
@Override
|
59
|
+
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
60
|
+
{
|
61
|
+
|
62
|
+
/*
|
63
|
+
* available mappings
|
64
|
+
*
|
65
|
+
* boolean -> unused
|
66
|
+
* byte -> unused
|
67
|
+
* short -> unused
|
68
|
+
* int -> NUMBER
|
69
|
+
* long -> NUMBER
|
70
|
+
* BigDecimal -> NUMBER
|
71
|
+
* String -> CHAR,VARCHAR,LONGVARCHAR,CLOB,NCHAR,NVARCHAR,NCLOB,NUMBER
|
72
|
+
* NString -> unused
|
73
|
+
* bytes -> unused
|
74
|
+
* SqlDate -> unused
|
75
|
+
* SqlTime -> unused
|
76
|
+
* SqlTimeStamp -> TIMESTAMP
|
77
|
+
*
|
78
|
+
*/
|
79
|
+
|
80
|
+
formats = new DateFormat[insertSchema.getCount()];
|
81
|
+
List<ColumnDefinition> columns = new ArrayList<ColumnDefinition>();
|
82
|
+
java.sql.Timestamp dummy = new java.sql.Timestamp(System.currentTimeMillis());
|
83
|
+
for (int i = 0; i < insertSchema.getCount(); i++) {
|
84
|
+
JdbcColumn insertColumn = insertSchema.getColumn(i);
|
85
|
+
switch (insertColumn.getSqlType()) {
|
86
|
+
case Types.CHAR:
|
87
|
+
case Types.VARCHAR:
|
88
|
+
case Types.LONGVARCHAR:
|
89
|
+
case Types.CLOB:
|
90
|
+
// TODO: CHAR(n CHAR)
|
91
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
92
|
+
ColumnDefinition.SQLT_CHR,
|
93
|
+
insertColumn.getSizeTypeParameter()));
|
94
|
+
break;
|
95
|
+
|
96
|
+
case Types.DECIMAL:
|
97
|
+
// sign + size
|
98
|
+
int size = 1 + insertColumn.getSizeTypeParameter();
|
99
|
+
if (insertColumn.getSizeTypeParameter() > 0) {
|
100
|
+
// decimal point
|
101
|
+
size += 1;
|
102
|
+
}
|
103
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
104
|
+
ColumnDefinition.SQLT_CHR,
|
105
|
+
size));
|
106
|
+
break;
|
107
|
+
|
108
|
+
case Types.DATE:
|
109
|
+
break;
|
110
|
+
|
111
|
+
case Types.TIMESTAMP:
|
112
|
+
String oracleFormat;
|
113
|
+
DateFormat javaFormat;
|
114
|
+
if (insertColumn.getSimpleTypeName().equals("DATE")) {
|
115
|
+
oracleFormat = "YYYY-MM-DD HH24:MI:SS";
|
116
|
+
javaFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
117
|
+
} else {
|
118
|
+
oracleFormat = "YYYY-MM-DD HH24:MI:SS.FF9";
|
119
|
+
javaFormat = new TimestampFormat("yyyy-MM-dd HH:mm:ss", 9);
|
120
|
+
}
|
121
|
+
formats[i] = javaFormat;
|
122
|
+
columns.add(new ColumnDefinition(insertColumn.getName(),
|
123
|
+
ColumnDefinition.SQLT_CHR,
|
124
|
+
javaFormat.format(dummy).length(),
|
125
|
+
oracleFormat));
|
126
|
+
break;
|
127
|
+
|
128
|
+
default:
|
129
|
+
throw new SQLException("Unsupported type : " + insertColumn.getSimpleTypeName());
|
130
|
+
}
|
131
|
+
|
132
|
+
}
|
133
|
+
|
134
|
+
rowSize = 0;
|
135
|
+
for (ColumnDefinition column : columns) {
|
136
|
+
rowSize += column.columnSize;
|
137
|
+
}
|
138
|
+
|
139
|
+
TableDefinition tableDefinition = new TableDefinition(table, charset.getId(), columns);
|
140
|
+
ociManager.open(ociKey, database, user, password, tableDefinition);
|
141
|
+
|
142
|
+
buffer = new RowBuffer(tableDefinition, Math.max(batchSize / rowSize, 8), charset.getJavaCharset());
|
143
|
+
}
|
144
|
+
|
145
|
+
@Override
|
146
|
+
public int getBatchWeight()
|
147
|
+
{
|
148
|
+
return batchWeight;
|
149
|
+
}
|
150
|
+
|
151
|
+
@Override
|
152
|
+
public void add() throws IOException, SQLException
|
153
|
+
{
|
154
|
+
batchWeight += rowSize;
|
155
|
+
if (buffer.isFull()) {
|
156
|
+
flush();
|
157
|
+
}
|
158
|
+
}
|
159
|
+
|
160
|
+
@Override
|
161
|
+
public void close() throws IOException, SQLException
|
162
|
+
{
|
163
|
+
ociManager.close(ociKey);
|
164
|
+
}
|
165
|
+
|
166
|
+
@Override
|
167
|
+
public void flush() throws IOException, SQLException
|
168
|
+
{
|
169
|
+
if (buffer.getRowCount() > 0) {
|
170
|
+
try {
|
171
|
+
logger.info(String.format("Loading %,d rows", buffer.getRowCount()));
|
172
|
+
|
173
|
+
long startTime = System.currentTimeMillis();
|
174
|
+
|
175
|
+
OCIWrapper oci = ociManager.get(ociKey);
|
176
|
+
synchronized (oci) {
|
177
|
+
oci.loadBuffer(buffer.getBuffer(), buffer.getRowCount());
|
178
|
+
}
|
179
|
+
|
180
|
+
totalRows += buffer.getRowCount();
|
181
|
+
double seconds = (System.currentTimeMillis() - startTime) / 1000.0;
|
182
|
+
logger.info(String.format("> %.2f seconds (loaded %,d rows in total)", seconds, totalRows));
|
183
|
+
|
184
|
+
} finally {
|
185
|
+
buffer.clear();
|
186
|
+
batchWeight = 0;
|
187
|
+
}
|
188
|
+
}
|
189
|
+
}
|
190
|
+
|
191
|
+
@Override
|
192
|
+
public void finish() throws IOException, SQLException
|
193
|
+
{
|
194
|
+
flush();
|
195
|
+
}
|
196
|
+
|
197
|
+
@Override
|
198
|
+
public void setNull(int sqlType) throws IOException, SQLException
|
199
|
+
{
|
200
|
+
buffer.addValue("");
|
201
|
+
}
|
202
|
+
|
203
|
+
@Override
|
204
|
+
public void setBoolean(boolean v) throws IOException, SQLException
|
205
|
+
{
|
206
|
+
throw new SQLException("Unsupported");
|
207
|
+
}
|
208
|
+
|
209
|
+
@Override
|
210
|
+
public void setByte(byte v) throws IOException, SQLException
|
211
|
+
{
|
212
|
+
throw new SQLException("Unsupported");
|
213
|
+
}
|
214
|
+
|
215
|
+
@Override
|
216
|
+
public void setShort(short v) throws IOException, SQLException
|
217
|
+
{
|
218
|
+
throw new SQLException("Unsupported");
|
219
|
+
}
|
220
|
+
|
221
|
+
@Override
|
222
|
+
public void setInt(int v) throws IOException, SQLException
|
223
|
+
{
|
224
|
+
buffer.addValue(v);
|
225
|
+
}
|
226
|
+
|
227
|
+
@Override
|
228
|
+
public void setLong(long v) throws IOException, SQLException
|
229
|
+
{
|
230
|
+
buffer.addValue(Long.toString(v));
|
231
|
+
}
|
232
|
+
|
233
|
+
@Override
|
234
|
+
public void setFloat(float v) throws IOException, SQLException
|
235
|
+
{
|
236
|
+
throw new SQLException("Unsupported");
|
237
|
+
}
|
238
|
+
|
239
|
+
@Override
|
240
|
+
public void setDouble(double v) throws IOException, SQLException
|
241
|
+
{
|
242
|
+
throw new SQLException("Unsupported");
|
243
|
+
}
|
244
|
+
|
245
|
+
@Override
|
246
|
+
public void setBigDecimal(BigDecimal v) throws IOException, SQLException
|
247
|
+
{
|
248
|
+
buffer.addValue(v);
|
249
|
+
}
|
250
|
+
|
251
|
+
@Override
|
252
|
+
public void setString(String v) throws IOException, SQLException
|
253
|
+
{
|
254
|
+
buffer.addValue(v);
|
255
|
+
}
|
256
|
+
|
257
|
+
@Override
|
258
|
+
public void setNString(String v) throws IOException, SQLException
|
259
|
+
{
|
260
|
+
throw new SQLException("Unsupported");
|
261
|
+
}
|
262
|
+
|
263
|
+
@Override
|
264
|
+
public void setBytes(byte[] v) throws IOException, SQLException
|
265
|
+
{
|
266
|
+
throw new SQLException("Unsupported");
|
267
|
+
}
|
268
|
+
|
269
|
+
@Override
|
270
|
+
public void setSqlDate(Timestamp v, Calendar calendar) throws IOException, SQLException
|
271
|
+
{
|
272
|
+
throw new SQLException("Unsupported");
|
273
|
+
}
|
274
|
+
|
275
|
+
@Override
|
276
|
+
public void setSqlTime(Timestamp v, Calendar calendar) throws IOException, SQLException
|
277
|
+
{
|
278
|
+
throw new SQLException("Unsupported");
|
279
|
+
}
|
280
|
+
|
281
|
+
@Override
|
282
|
+
public void setSqlTimestamp(Timestamp v, Calendar calendar) throws IOException, SQLException
|
283
|
+
{
|
284
|
+
java.sql.Timestamp t = new java.sql.Timestamp(v.toEpochMilli());
|
285
|
+
t.setNanos(v.getNano());
|
286
|
+
DateFormat format = formats[buffer.getCurrentColumn()];
|
287
|
+
format.setTimeZone(calendar.getTimeZone());
|
288
|
+
buffer.addValue(format.format(t));
|
289
|
+
}
|
290
|
+
}
|