embulk-output-oracle 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +46 -0
- data/build.gradle +3 -0
- data/classpath/embulk-output-jdbc-0.2.1.jar +0 -0
- data/classpath/embulk-output-oracle-0.2.1.jar +0 -0
- data/lib/embulk/output/oracle.rb +3 -0
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +151 -0
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +75 -0
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +47 -0
- data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java +31 -0
- metadata +53 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: cc81277bd2a1f2ae80924bb543d6eecfcd3d4d6c
|
4
|
+
data.tar.gz: 3b0b5fa66bc6429a37b1ce15bd3a1d9ea4aca237
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 215e1945daf0792754efdfb7c6adc1596d9a020a8dd53584367be82f9a9d44397a77ae42cbd2601f78b1290e4d2afab22a05c9c6a3fd6e47e6c900dad3484231
|
7
|
+
data.tar.gz: 4078b17328554d4abe9bbfa89aab60d109fe6acd29c7b1c1011068375a6e879a7afe6b4c18ac34b4b0e5581be2a875dc4485c320f8aa032362543362b8e762f3
|
data/README.md
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# Oracle output plugins for Embulk
|
2
|
+
|
3
|
+
Oracle output plugins for Embulk loads records to Oracle.
|
4
|
+
|
5
|
+
## Overview
|
6
|
+
|
7
|
+
* **Plugin type**: output
|
8
|
+
* **Load all or nothing**: depnds on the mode:
|
9
|
+
* **insert**: no
|
10
|
+
* **replace**: yes
|
11
|
+
* **Resume supported**: no
|
12
|
+
|
13
|
+
## Configuration
|
14
|
+
|
15
|
+
- **driver_path**: path to the jar file of the Oracle JDBC driver (string)
|
16
|
+
- **host**: database host name (string, required if url is not set)
|
17
|
+
- **port**: database port number (integer, default: 1521)
|
18
|
+
- **user**: database login user name (string, required)
|
19
|
+
- **password**: database login password (string, default: "")
|
20
|
+
- **database**: destination database name (string, required if url is not set)
|
21
|
+
- **url**: URL of the JDBC connection (string, optional)
|
22
|
+
- **table**: destination table name (string, required)
|
23
|
+
- **mode**: "replace" or "insert" (string, required)
|
24
|
+
- **batch_size**: size of a single batch insert (integer, default: 16777216)
|
25
|
+
- **options**: extra connection properties (hash, default: {})
|
26
|
+
|
27
|
+
|
28
|
+
### Example
|
29
|
+
|
30
|
+
```yaml
|
31
|
+
out:
|
32
|
+
type: oracle
|
33
|
+
driver_path: /opt/oracle/ojdbc6.jar
|
34
|
+
host: localhost
|
35
|
+
user: root
|
36
|
+
password: ""
|
37
|
+
database: my_database
|
38
|
+
table: my_table
|
39
|
+
mode: insert
|
40
|
+
```
|
41
|
+
|
42
|
+
### Build
|
43
|
+
|
44
|
+
```
|
45
|
+
$ ./gradlew gem
|
46
|
+
```
|
data/build.gradle
ADDED
Binary file
|
Binary file
|
@@ -0,0 +1,151 @@
|
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.sql.SQLException;
|
5
|
+
import java.util.Properties;
|
6
|
+
import com.google.common.base.Optional;
|
7
|
+
import org.embulk.config.Config;
|
8
|
+
import org.embulk.config.ConfigException;
|
9
|
+
import org.embulk.config.ConfigDefault;
|
10
|
+
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
11
|
+
import org.embulk.output.jdbc.BatchInsert;
|
12
|
+
import org.embulk.output.jdbc.StandardBatchInsert;
|
13
|
+
import org.embulk.output.jdbc.setter.ColumnSetterFactory;
|
14
|
+
import org.embulk.output.oracle.OracleOutputConnection;
|
15
|
+
import org.embulk.output.oracle.OracleOutputConnector;
|
16
|
+
import org.embulk.output.oracle.setter.OracleColumnSetterFactory;
|
17
|
+
import org.embulk.spi.PageReader;
|
18
|
+
import org.embulk.spi.time.TimestampFormatter;
|
19
|
+
|
20
|
+
public class OracleOutputPlugin
|
21
|
+
extends AbstractJdbcOutputPlugin
|
22
|
+
{
|
23
|
+
private static final int MAX_TABLE_NAME_LENGTH = 30;
|
24
|
+
|
25
|
+
public interface OraclePluginTask
|
26
|
+
extends PluginTask
|
27
|
+
{
|
28
|
+
@Config("driver_path")
|
29
|
+
@ConfigDefault("null")
|
30
|
+
public Optional<String> getDriverPath();
|
31
|
+
|
32
|
+
@Config("host")
|
33
|
+
@ConfigDefault("null")
|
34
|
+
public Optional<String> getHost();
|
35
|
+
|
36
|
+
@Config("port")
|
37
|
+
@ConfigDefault("1521")
|
38
|
+
public int getPort();
|
39
|
+
|
40
|
+
@Config("database")
|
41
|
+
@ConfigDefault("null")
|
42
|
+
public Optional<String> getDatabase();
|
43
|
+
|
44
|
+
@Config("url")
|
45
|
+
@ConfigDefault("null")
|
46
|
+
public Optional<String> getUrl();
|
47
|
+
|
48
|
+
@Config("user")
|
49
|
+
public String getUser();
|
50
|
+
|
51
|
+
@Config("password")
|
52
|
+
@ConfigDefault("\"\"")
|
53
|
+
public String getPassword();
|
54
|
+
}
|
55
|
+
|
56
|
+
@Override
|
57
|
+
protected Class<? extends PluginTask> getTaskClass()
|
58
|
+
{
|
59
|
+
return OraclePluginTask.class;
|
60
|
+
}
|
61
|
+
|
62
|
+
@Override
|
63
|
+
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
64
|
+
{
|
65
|
+
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
66
|
+
|
67
|
+
if (oracleTask.getDriverPath().isPresent()) {
|
68
|
+
loadDriverJar(oracleTask.getDriverPath().get());
|
69
|
+
}
|
70
|
+
|
71
|
+
String url;
|
72
|
+
if (oracleTask.getUrl().isPresent()) {
|
73
|
+
if (oracleTask.getHost().isPresent() || oracleTask.getDatabase().isPresent()) {
|
74
|
+
throw new IllegalArgumentException("'host', 'port' and 'database' parameters are invalid if 'url' parameter is set.");
|
75
|
+
}
|
76
|
+
|
77
|
+
url = oracleTask.getUrl().get();
|
78
|
+
} else {
|
79
|
+
if (!oracleTask.getHost().isPresent()) {
|
80
|
+
throw new IllegalArgumentException("Field 'host' is not set.");
|
81
|
+
}
|
82
|
+
if (!oracleTask.getDatabase().isPresent()) {
|
83
|
+
throw new IllegalArgumentException("Field 'database' is not set.");
|
84
|
+
}
|
85
|
+
|
86
|
+
url = String.format("jdbc:oracle:thin:@%s:%d:%s",
|
87
|
+
oracleTask.getHost().get(), oracleTask.getPort(), oracleTask.getDatabase().get());
|
88
|
+
}
|
89
|
+
|
90
|
+
Properties props = new Properties();
|
91
|
+
props.setProperty("user", oracleTask.getUser());
|
92
|
+
props.setProperty("password", oracleTask.getPassword());
|
93
|
+
props.putAll(oracleTask.getOptions());
|
94
|
+
|
95
|
+
return new OracleOutputConnector(url, props);
|
96
|
+
}
|
97
|
+
|
98
|
+
@Override
|
99
|
+
protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
|
100
|
+
{
|
101
|
+
return new StandardBatchInsert(getConnector(task, true));
|
102
|
+
}
|
103
|
+
|
104
|
+
@Override
|
105
|
+
protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
|
106
|
+
TimestampFormatter timestampFormatter)
|
107
|
+
{
|
108
|
+
return new OracleColumnSetterFactory(batch, pageReader, timestampFormatter);
|
109
|
+
}
|
110
|
+
|
111
|
+
@Override
|
112
|
+
protected String generateSwapTableName(PluginTask task) throws SQLException
|
113
|
+
{
|
114
|
+
return generateSwapTableName(task, "_bl_tmp", MAX_TABLE_NAME_LENGTH);
|
115
|
+
}
|
116
|
+
|
117
|
+
// TODO move this method to AbstractJdbcOutputPlugin
|
118
|
+
protected String generateSwapTableName(PluginTask task, String suffix, int maxTableNameLength) throws SQLException
|
119
|
+
{
|
120
|
+
String tableName = task.getTable();
|
121
|
+
String uniqueSuffix = getTransactionUniqueName() + suffix;
|
122
|
+
|
123
|
+
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) { // + 1 for '_'
|
124
|
+
// truncate transaction unique name
|
125
|
+
int suffixLength = Math.max(maxTableNameLength - tableName.length() - 1, suffix.length() + 8); // include 8 characters of the transaction name at least
|
126
|
+
uniqueSuffix = uniqueSuffix.substring(uniqueSuffix.length() - suffixLength);
|
127
|
+
}
|
128
|
+
|
129
|
+
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {
|
130
|
+
// use truncated table name
|
131
|
+
int truncLength = maxTableNameLength - uniqueSuffix.length() - 1;
|
132
|
+
while (true) {
|
133
|
+
truncLength--;
|
134
|
+
if (truncLength <= 0) {
|
135
|
+
throw new ConfigException("Table name is too long to generate temporary table name");
|
136
|
+
}
|
137
|
+
tableName = tableName.substring(0, truncLength);
|
138
|
+
//if (!connection.tableExists(tableName)) {
|
139
|
+
// TODO this doesn't help. Rather than truncating more characters,
|
140
|
+
// here needs to replace characters with random characters. But
|
141
|
+
// to make the result deterministic. So, an idea is replacing
|
142
|
+
// the last character to the first (second, third, ... for each loop)
|
143
|
+
// of md5(original table name).
|
144
|
+
return tableName + "_" + uniqueSuffix;
|
145
|
+
//}
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
149
|
+
return tableName + "_" + uniqueSuffix;
|
150
|
+
}
|
151
|
+
}
|
@@ -0,0 +1,75 @@
|
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
|
4
|
+
import java.sql.Connection;
|
5
|
+
import java.sql.ResultSet;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import java.sql.Statement;
|
8
|
+
|
9
|
+
import org.embulk.output.jdbc.JdbcOutputConnection;
|
10
|
+
import org.embulk.output.jdbc.JdbcSchema;
|
11
|
+
|
12
|
+
public class OracleOutputConnection
|
13
|
+
extends JdbcOutputConnection
|
14
|
+
{
|
15
|
+
public OracleOutputConnection(Connection connection, boolean autoCommit)
|
16
|
+
throws SQLException
|
17
|
+
{
|
18
|
+
super(connection, getSchema(connection));
|
19
|
+
connection.setAutoCommit(autoCommit);
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
protected String convertTypeName(String typeName)
|
24
|
+
{
|
25
|
+
switch(typeName) {
|
26
|
+
case "BIGINT":
|
27
|
+
return "NUMBER(19,0)";
|
28
|
+
default:
|
29
|
+
return typeName;
|
30
|
+
}
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
protected void setSearchPath(String schema) throws SQLException {
|
35
|
+
// NOP
|
36
|
+
}
|
37
|
+
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public void dropTableIfExists(String tableName) throws SQLException
|
41
|
+
{
|
42
|
+
if (tableExists(tableName)) {
|
43
|
+
dropTable(tableName);
|
44
|
+
}
|
45
|
+
}
|
46
|
+
|
47
|
+
@Override
|
48
|
+
protected void dropTableIfExists(Statement stmt, String tableName) throws SQLException {
|
49
|
+
if (tableExists(tableName)) {
|
50
|
+
dropTable(stmt, tableName);
|
51
|
+
}
|
52
|
+
}
|
53
|
+
|
54
|
+
@Override
|
55
|
+
public void createTableIfNotExists(String tableName, JdbcSchema schema) throws SQLException
|
56
|
+
{
|
57
|
+
if (!tableExists(tableName)) {
|
58
|
+
createTable(tableName, schema);
|
59
|
+
}
|
60
|
+
}
|
61
|
+
|
62
|
+
private static String getSchema(Connection connection) throws SQLException
|
63
|
+
{
|
64
|
+
// Because old Oracle JDBC drivers don't support Connection#getSchema method.
|
65
|
+
String sql = "SELECT SYS_CONTEXT('USERENV', 'CURRENT_SCHEMA') FROM DUAL";
|
66
|
+
try (Statement statement = connection.createStatement()) {
|
67
|
+
try (ResultSet resultSet = statement.executeQuery(sql)) {
|
68
|
+
if (resultSet.next()) {
|
69
|
+
return resultSet.getString(1);
|
70
|
+
}
|
71
|
+
throw new SQLException(String.format("Cannot get schema becase \"%s\" didn't return any value.", sql));
|
72
|
+
}
|
73
|
+
}
|
74
|
+
}
|
75
|
+
}
|
@@ -0,0 +1,47 @@
|
|
1
|
+
package org.embulk.output.oracle;
|
2
|
+
|
3
|
+
import java.sql.Connection;
|
4
|
+
import java.sql.DriverManager;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import java.util.Properties;
|
7
|
+
|
8
|
+
import org.embulk.output.jdbc.JdbcOutputConnector;
|
9
|
+
|
10
|
+
public class OracleOutputConnector
|
11
|
+
implements JdbcOutputConnector
|
12
|
+
{
|
13
|
+
private final String url;
|
14
|
+
private final Properties properties;
|
15
|
+
|
16
|
+
public OracleOutputConnector(String url, Properties properties)
|
17
|
+
{
|
18
|
+
try {
|
19
|
+
Class.forName("oracle.jdbc.OracleDriver");
|
20
|
+
} catch (Exception ex) {
|
21
|
+
throw new RuntimeException(ex);
|
22
|
+
}
|
23
|
+
this.url = url;
|
24
|
+
this.properties = properties;
|
25
|
+
}
|
26
|
+
|
27
|
+
@Override
|
28
|
+
public OracleOutputConnection connect(boolean autoCommit) throws SQLException
|
29
|
+
{
|
30
|
+
Connection c = DriverManager.getConnection(url, properties);
|
31
|
+
if (c == null) {
|
32
|
+
// driver.connect returns null when url is "jdbc:mysql://...".
|
33
|
+
throw new SQLException("Invalid url : " + url);
|
34
|
+
}
|
35
|
+
|
36
|
+
try {
|
37
|
+
OracleOutputConnection con = new OracleOutputConnection(c, autoCommit);
|
38
|
+
c = null;
|
39
|
+
return con;
|
40
|
+
|
41
|
+
} finally {
|
42
|
+
if (c != null) {
|
43
|
+
c.close();
|
44
|
+
}
|
45
|
+
}
|
46
|
+
}
|
47
|
+
}
|
@@ -0,0 +1,31 @@
|
|
1
|
+
package org.embulk.output.oracle.setter;
|
2
|
+
|
3
|
+
import java.sql.Types;
|
4
|
+
|
5
|
+
import org.embulk.output.jdbc.BatchInsert;
|
6
|
+
import org.embulk.output.jdbc.JdbcColumn;
|
7
|
+
import org.embulk.output.jdbc.setter.ColumnSetter;
|
8
|
+
import org.embulk.output.jdbc.setter.ColumnSetterFactory;
|
9
|
+
import org.embulk.output.jdbc.setter.StringColumnSetter;
|
10
|
+
import org.embulk.spi.PageReader;
|
11
|
+
import org.embulk.spi.time.TimestampFormatter;
|
12
|
+
|
13
|
+
public class OracleColumnSetterFactory extends ColumnSetterFactory
|
14
|
+
{
|
15
|
+
public OracleColumnSetterFactory(BatchInsert batch, PageReader pageReader,
|
16
|
+
TimestampFormatter timestampFormatter)
|
17
|
+
{
|
18
|
+
super(batch, pageReader, timestampFormatter);
|
19
|
+
}
|
20
|
+
|
21
|
+
@Override
|
22
|
+
public ColumnSetter newColumnSetter(JdbcColumn column)
|
23
|
+
{
|
24
|
+
switch (column.getSqlType()) {
|
25
|
+
case Types.DECIMAL:
|
26
|
+
return new StringColumnSetter(batch, pageReader, column, timestampFormatter);
|
27
|
+
default:
|
28
|
+
return super.newColumnSetter(column);
|
29
|
+
}
|
30
|
+
}
|
31
|
+
}
|
metadata
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: embulk-output-oracle
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Sadayuki Furuhashi
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2015-03-15 00:00:00.000000000 Z
|
12
|
+
dependencies: []
|
13
|
+
description: Inserts or updates records to a table.
|
14
|
+
email:
|
15
|
+
- frsyuki@gmail.com
|
16
|
+
executables: []
|
17
|
+
extensions: []
|
18
|
+
extra_rdoc_files: []
|
19
|
+
files:
|
20
|
+
- README.md
|
21
|
+
- build.gradle
|
22
|
+
- lib/embulk/output/oracle.rb
|
23
|
+
- src/main/java/org/embulk/output/OracleOutputPlugin.java
|
24
|
+
- src/main/java/org/embulk/output/oracle/OracleOutputConnection.java
|
25
|
+
- src/main/java/org/embulk/output/oracle/OracleOutputConnector.java
|
26
|
+
- src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java
|
27
|
+
- classpath/embulk-output-jdbc-0.2.1.jar
|
28
|
+
- classpath/embulk-output-oracle-0.2.1.jar
|
29
|
+
homepage: https://github.com/embulk/embulk-output-jdbc
|
30
|
+
licenses:
|
31
|
+
- Apache 2.0
|
32
|
+
metadata: {}
|
33
|
+
post_install_message:
|
34
|
+
rdoc_options: []
|
35
|
+
require_paths:
|
36
|
+
- lib
|
37
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - '>='
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '0'
|
42
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - '>='
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
requirements: []
|
48
|
+
rubyforge_project:
|
49
|
+
rubygems_version: 2.1.9
|
50
|
+
signing_key:
|
51
|
+
specification_version: 4
|
52
|
+
summary: JDBC output plugin for Embulk
|
53
|
+
test_files: []
|