embulk-output-oracle 0.4.1 → 0.4.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +160 -160
- data/build.gradle +6 -6
- data/classpath/embulk-output-jdbc-0.4.2.jar +0 -0
- data/classpath/{embulk-output-oracle-0.4.1.jar → embulk-output-oracle-0.4.2.jar} +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -153
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +290 -290
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -32
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +179 -165
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +105 -105
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +185 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +50 -25
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +548 -540
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/java/org/embulk/output/tester/EmbulkPluginTester.java +79 -0
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/dummy.txt +1 -0
- data/src/test/resources/yml/test-insert-direct-direct-method.yml +26 -26
- data/src/test/resources/yml/test-insert-direct-empty.yml +25 -25
- data/src/test/resources/yml/test-insert-direct-oci-method-split.yml +28 -28
- data/src/test/resources/yml/test-insert-direct-oci-method.yml +28 -28
- data/src/test/resources/yml/test-insert-direct.yml +25 -25
- data/src/test/resources/yml/test-insert-empty.yml +27 -27
- data/src/test/resources/yml/test-insert.yml +25 -27
- data/src/test/resources/yml/test-replace-empty.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +31 -31
- data/src/test/resources/yml/test-replace-long-name.yml +31 -31
- data/src/test/resources/yml/test-replace.yml +31 -31
- data/src/test/resources/yml/test-string-timestamp.yml +28 -28
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +6 -8
- data/classpath/embulk-output-jdbc-0.4.1.jar +0 -0
- data/lib/embulk/linux_x64/libembulk-output-oracle.so +0 -0
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +0 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +0 -124
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +0 -105
@@ -1,8 +1,8 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
public enum InsertMethod
|
4
|
-
{
|
5
|
-
normal,
|
6
|
-
direct,
|
7
|
-
oci;
|
8
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
public enum InsertMethod
|
4
|
+
{
|
5
|
+
normal,
|
6
|
+
direct,
|
7
|
+
oci;
|
8
|
+
}
|
@@ -1,32 +1,32 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
import java.nio.charset.Charset;
|
4
|
-
|
5
|
-
public class OracleCharset
|
6
|
-
{
|
7
|
-
private final String name;
|
8
|
-
private final short id;
|
9
|
-
private final Charset javaCharset;
|
10
|
-
|
11
|
-
public OracleCharset(String name, short id, Charset javaCharset)
|
12
|
-
{
|
13
|
-
this.name = name;
|
14
|
-
this.id = id;
|
15
|
-
this.javaCharset = javaCharset;
|
16
|
-
}
|
17
|
-
|
18
|
-
public String getName()
|
19
|
-
{
|
20
|
-
return name;
|
21
|
-
}
|
22
|
-
|
23
|
-
public short getId()
|
24
|
-
{
|
25
|
-
return id;
|
26
|
-
}
|
27
|
-
|
28
|
-
public Charset getJavaCharset()
|
29
|
-
{
|
30
|
-
return javaCharset;
|
31
|
-
}
|
32
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.nio.charset.Charset;
|
4
|
+
|
5
|
+
public class OracleCharset
|
6
|
+
{
|
7
|
+
private final String name;
|
8
|
+
private final short id;
|
9
|
+
private final Charset javaCharset;
|
10
|
+
|
11
|
+
public OracleCharset(String name, short id, Charset javaCharset)
|
12
|
+
{
|
13
|
+
this.name = name;
|
14
|
+
this.id = id;
|
15
|
+
this.javaCharset = javaCharset;
|
16
|
+
}
|
17
|
+
|
18
|
+
public String getName()
|
19
|
+
{
|
20
|
+
return name;
|
21
|
+
}
|
22
|
+
|
23
|
+
public short getId()
|
24
|
+
{
|
25
|
+
return id;
|
26
|
+
}
|
27
|
+
|
28
|
+
public Charset getJavaCharset()
|
29
|
+
{
|
30
|
+
return javaCharset;
|
31
|
+
}
|
32
|
+
}
|
@@ -1,165 +1,179 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
import java.nio.charset.Charset;
|
4
|
-
import java.sql.Connection;
|
5
|
-
import java.sql.PreparedStatement;
|
6
|
-
import java.sql.ResultSet;
|
7
|
-
import java.sql.SQLException;
|
8
|
-
import java.sql.Statement;
|
9
|
-
import java.util.
|
10
|
-
import java.util.
|
11
|
-
|
12
|
-
|
13
|
-
import org.embulk.output.jdbc.
|
14
|
-
import org.embulk.output.jdbc.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
static
|
21
|
-
|
22
|
-
CHARSET_NAMES.put("
|
23
|
-
CHARSET_NAMES.put("
|
24
|
-
CHARSET_NAMES.put("
|
25
|
-
CHARSET_NAMES.put("
|
26
|
-
CHARSET_NAMES.put("
|
27
|
-
CHARSET_NAMES.put("
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
connection
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
sb
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.nio.charset.Charset;
|
4
|
+
import java.sql.Connection;
|
5
|
+
import java.sql.PreparedStatement;
|
6
|
+
import java.sql.ResultSet;
|
7
|
+
import java.sql.SQLException;
|
8
|
+
import java.sql.Statement;
|
9
|
+
import java.util.Arrays;
|
10
|
+
import java.util.HashMap;
|
11
|
+
import java.util.Map;
|
12
|
+
|
13
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
14
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
15
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
16
|
+
|
17
|
+
public class OracleOutputConnection
|
18
|
+
extends JdbcOutputConnection
|
19
|
+
{
|
20
|
+
private static final Map<String, String> CHARSET_NAMES = new HashMap<String, String>();
|
21
|
+
static {
|
22
|
+
CHARSET_NAMES.put("JA16SJIS", "MS932");
|
23
|
+
CHARSET_NAMES.put("JA16SJISTILDE", "MS932");
|
24
|
+
CHARSET_NAMES.put("JA16EUC", "EUC-JP");
|
25
|
+
CHARSET_NAMES.put("JA16EUCTILDE", "EUC-JP");
|
26
|
+
CHARSET_NAMES.put("AL32UTF8", "UTF-8");
|
27
|
+
CHARSET_NAMES.put("UTF8", "UTF-8");
|
28
|
+
CHARSET_NAMES.put("AL16UTF16", "UTF-16");
|
29
|
+
}
|
30
|
+
|
31
|
+
private final boolean direct;
|
32
|
+
private OracleCharset charset;
|
33
|
+
|
34
|
+
public OracleOutputConnection(Connection connection, boolean autoCommit, boolean direct)
|
35
|
+
throws SQLException
|
36
|
+
{
|
37
|
+
super(connection, getSchema(connection));
|
38
|
+
connection.setAutoCommit(autoCommit);
|
39
|
+
|
40
|
+
this.direct = direct;
|
41
|
+
}
|
42
|
+
|
43
|
+
@Override
|
44
|
+
protected String buildColumnTypeName(JdbcColumn c)
|
45
|
+
{
|
46
|
+
switch(c.getSimpleTypeName()) {
|
47
|
+
case "BIGINT":
|
48
|
+
return "NUMBER(19,0)";
|
49
|
+
default:
|
50
|
+
return super.buildColumnTypeName(c);
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
@Override
|
55
|
+
protected void setSearchPath(String schema) throws SQLException {
|
56
|
+
// NOP
|
57
|
+
}
|
58
|
+
|
59
|
+
@Override
|
60
|
+
public void dropTableIfExists(String tableName) throws SQLException
|
61
|
+
{
|
62
|
+
if (tableExists(tableName)) {
|
63
|
+
dropTable(tableName);
|
64
|
+
}
|
65
|
+
}
|
66
|
+
|
67
|
+
@Override
|
68
|
+
protected void dropTableIfExists(Statement stmt, String tableName) throws SQLException {
|
69
|
+
if (tableExists(tableName)) {
|
70
|
+
dropTable(stmt, tableName);
|
71
|
+
}
|
72
|
+
}
|
73
|
+
|
74
|
+
@Override
|
75
|
+
public void createTableIfNotExists(String tableName, JdbcSchema schema) throws SQLException
|
76
|
+
{
|
77
|
+
if (!tableExists(tableName)) {
|
78
|
+
createTable(tableName, schema);
|
79
|
+
}
|
80
|
+
}
|
81
|
+
|
82
|
+
public void createTable(String tableName, JdbcSchema schema) throws SQLException
|
83
|
+
{
|
84
|
+
Statement stmt = connection.createStatement();
|
85
|
+
try {
|
86
|
+
String sql = buildCreateTableSql(tableName, schema);
|
87
|
+
executeUpdate(stmt, sql);
|
88
|
+
commitIfNecessary(connection);
|
89
|
+
} catch (SQLException ex) {
|
90
|
+
throw safeRollback(connection, ex);
|
91
|
+
} finally {
|
92
|
+
stmt.close();
|
93
|
+
}
|
94
|
+
}
|
95
|
+
|
96
|
+
protected String buildCreateTableSql(String name, JdbcSchema schema)
|
97
|
+
{
|
98
|
+
StringBuilder sb = new StringBuilder();
|
99
|
+
|
100
|
+
sb.append("CREATE TABLE ");
|
101
|
+
quoteIdentifierString(sb, name);
|
102
|
+
sb.append(buildCreateTableSchemaSql(schema));
|
103
|
+
return sb.toString();
|
104
|
+
}
|
105
|
+
|
106
|
+
private static String getSchema(Connection connection) throws SQLException
|
107
|
+
{
|
108
|
+
// Because old Oracle JDBC drivers don't support Connection#getSchema method.
|
109
|
+
String sql = "SELECT SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') FROM DUAL";
|
110
|
+
try (Statement statement = connection.createStatement()) {
|
111
|
+
try (ResultSet resultSet = statement.executeQuery(sql)) {
|
112
|
+
if (resultSet.next()) {
|
113
|
+
return resultSet.getString(1);
|
114
|
+
}
|
115
|
+
throw new SQLException(String.format("Cannot get schema becase \"%s\" didn't return any value.", sql));
|
116
|
+
}
|
117
|
+
}
|
118
|
+
}
|
119
|
+
|
120
|
+
@Override
|
121
|
+
protected String buildPreparedInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
122
|
+
{
|
123
|
+
String sql = super.buildPreparedInsertSql(toTable, toTableSchema);
|
124
|
+
if (direct) {
|
125
|
+
sql = sql.replaceAll("^INSERT ", "INSERT /*+ APPEND_VALUES */ ");
|
126
|
+
}
|
127
|
+
return sql;
|
128
|
+
}
|
129
|
+
|
130
|
+
@Override
|
131
|
+
public Charset getTableNameCharset() throws SQLException
|
132
|
+
{
|
133
|
+
return getOracleCharset().getJavaCharset();
|
134
|
+
}
|
135
|
+
|
136
|
+
public synchronized OracleCharset getOracleCharset() throws SQLException
|
137
|
+
{
|
138
|
+
if (charset == null) {
|
139
|
+
String charsetName = "UTF8";
|
140
|
+
try (Statement statement = connection.createStatement()) {
|
141
|
+
try (ResultSet resultSet = statement.executeQuery("SELECT VALUE FROM NLS_DATABASE_PARAMETERS WHERE PARAMETER='NLS_CHARACTERSET'")) {
|
142
|
+
if (resultSet.next()) {
|
143
|
+
String nlsCharacterSet = resultSet.getString(1);
|
144
|
+
if (CHARSET_NAMES.containsKey(nlsCharacterSet)) {
|
145
|
+
charsetName = nlsCharacterSet;
|
146
|
+
}
|
147
|
+
}
|
148
|
+
}
|
149
|
+
}
|
150
|
+
|
151
|
+
try (PreparedStatement statement = connection.prepareStatement("SELECT NLS_CHARSET_ID(?) FROM DUAL")) {
|
152
|
+
statement.setString(1, charsetName);
|
153
|
+
try (ResultSet resultSet = statement.executeQuery()) {
|
154
|
+
if (!resultSet.next()) {
|
155
|
+
throw new SQLException("Unknown NLS_CHARACTERSET : " + charsetName);
|
156
|
+
}
|
157
|
+
|
158
|
+
charset = new OracleCharset(charsetName,
|
159
|
+
resultSet.getShort(1),
|
160
|
+
Charset.forName(CHARSET_NAMES.get(charsetName)));
|
161
|
+
}
|
162
|
+
}
|
163
|
+
}
|
164
|
+
return charset;
|
165
|
+
}
|
166
|
+
|
167
|
+
private static final String[] STANDARD_SIZE_TYPE_NAMES = {
|
168
|
+
"VARCHAR2", "NVARCHAR2",
|
169
|
+
};
|
170
|
+
|
171
|
+
@Override
|
172
|
+
protected ColumnDeclareType getColumnDeclareType(String convertedTypeName, JdbcColumn col)
|
173
|
+
{
|
174
|
+
if (Arrays.asList(STANDARD_SIZE_TYPE_NAMES).contains(convertedTypeName)) {
|
175
|
+
return ColumnDeclareType.SIZE;
|
176
|
+
}
|
177
|
+
return super.getColumnDeclareType(convertedTypeName, col);
|
178
|
+
}
|
179
|
+
}
|
@@ -1,49 +1,49 @@
|
|
1
|
-
package org.embulk.output.oracle;
|
2
|
-
|
3
|
-
import java.sql.Connection;
|
4
|
-
import java.sql.DriverManager;
|
5
|
-
import java.sql.SQLException;
|
6
|
-
import java.util.Properties;
|
7
|
-
|
8
|
-
import org.embulk.output.jdbc.JdbcOutputConnector;
|
9
|
-
|
10
|
-
public class OracleOutputConnector
|
11
|
-
implements JdbcOutputConnector
|
12
|
-
{
|
13
|
-
private final String url;
|
14
|
-
private final Properties properties;
|
15
|
-
private final boolean direct;
|
16
|
-
|
17
|
-
public OracleOutputConnector(String url, Properties properties, boolean direct)
|
18
|
-
{
|
19
|
-
try {
|
20
|
-
Class.forName("oracle.jdbc.OracleDriver");
|
21
|
-
} catch (Exception ex) {
|
22
|
-
throw new RuntimeException(ex);
|
23
|
-
}
|
24
|
-
this.url = url;
|
25
|
-
this.properties = properties;
|
26
|
-
this.direct = direct;
|
27
|
-
}
|
28
|
-
|
29
|
-
@Override
|
30
|
-
public OracleOutputConnection connect(boolean autoCommit) throws SQLException
|
31
|
-
{
|
32
|
-
Connection c = DriverManager.getConnection(url, properties);
|
33
|
-
if (c == null) {
|
34
|
-
// driver.connect returns null when url is "jdbc:mysql://...".
|
35
|
-
throw new SQLException("Invalid url : " + url);
|
36
|
-
}
|
37
|
-
|
38
|
-
try {
|
39
|
-
OracleOutputConnection con = new OracleOutputConnection(c, autoCommit, direct);
|
40
|
-
c = null;
|
41
|
-
return con;
|
42
|
-
|
43
|
-
} finally {
|
44
|
-
if (c != null) {
|
45
|
-
c.close();
|
46
|
-
}
|
47
|
-
}
|
48
|
-
}
|
49
|
-
}
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.sql.Connection;
|
4
|
+
import java.sql.DriverManager;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import java.util.Properties;
|
7
|
+
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
9
|
+
|
10
|
+
public class OracleOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private final String url;
|
14
|
+
private final Properties properties;
|
15
|
+
private final boolean direct;
|
16
|
+
|
17
|
+
public OracleOutputConnector(String url, Properties properties, boolean direct)
|
18
|
+
{
|
19
|
+
try {
|
20
|
+
Class.forName("oracle.jdbc.OracleDriver");
|
21
|
+
} catch (Exception ex) {
|
22
|
+
throw new RuntimeException(ex);
|
23
|
+
}
|
24
|
+
this.url = url;
|
25
|
+
this.properties = properties;
|
26
|
+
this.direct = direct;
|
27
|
+
}
|
28
|
+
|
29
|
+
@Override
|
30
|
+
public OracleOutputConnection connect(boolean autoCommit) throws SQLException
|
31
|
+
{
|
32
|
+
Connection c = DriverManager.getConnection(url, properties);
|
33
|
+
if (c == null) {
|
34
|
+
// driver.connect returns null when url is "jdbc:mysql://...".
|
35
|
+
throw new SQLException("Invalid url : " + url);
|
36
|
+
}
|
37
|
+
|
38
|
+
try {
|
39
|
+
OracleOutputConnection con = new OracleOutputConnection(c, autoCommit, direct);
|
40
|
+
c = null;
|
41
|
+
return con;
|
42
|
+
|
43
|
+
} finally {
|
44
|
+
if (c != null) {
|
45
|
+
c.close();
|
46
|
+
}
|
47
|
+
}
|
48
|
+
}
|
49
|
+
}
|