embulk-output-oracle 0.2.4 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +155 -110
- data/build.gradle +6 -6
- data/classpath/embulk-output-jdbc-0.3.0.jar +0 -0
- data/classpath/embulk-output-oracle-0.3.0.jar +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +22 -66
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +289 -289
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -19
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -134
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +99 -99
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +120 -120
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +100 -100
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +172 -161
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +445 -413
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/yml/test-insert-direct.yml +26 -26
- data/src/test/resources/yml/test-insert-oci-split.yml +26 -26
- data/src/test/resources/yml/test-insert-oci.yml +26 -26
- data/src/test/resources/yml/test-insert.yml +25 -25
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +25 -0
- data/src/test/resources/yml/test-replace-long-name.yml +25 -25
- data/src/test/resources/yml/test-replace.yml +25 -25
- data/src/test/resources/yml/test-string-timestamp.yml +25 -0
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +6 -5
- data/classpath/embulk-output-jdbc-0.2.4.jar +0 -0
- data/classpath/embulk-output-oracle-0.2.4.jar +0 -0
- data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java +0 -31
@@ -1,35 +1,29 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Properties;
|
3
5
|
import java.io.IOException;
|
4
|
-
import java.
|
6
|
+
import java.nio.ByteBuffer;
|
5
7
|
import java.sql.SQLException;
|
6
|
-
import
|
7
|
-
|
8
|
+
import com.google.common.base.Optional;
|
9
|
+
import com.google.common.collect.ImmutableSet;
|
8
10
|
import org.embulk.config.Config;
|
9
11
|
import org.embulk.config.ConfigDefault;
|
10
|
-
import org.embulk.config.ConfigException;
|
11
12
|
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
12
13
|
import org.embulk.output.jdbc.BatchInsert;
|
13
14
|
import org.embulk.output.jdbc.JdbcOutputConnection;
|
14
15
|
import org.embulk.output.jdbc.JdbcOutputConnector;
|
15
16
|
import org.embulk.output.jdbc.StandardBatchInsert;
|
16
|
-
import org.embulk.output.jdbc.setter.ColumnSetterFactory;
|
17
17
|
import org.embulk.output.oracle.DirectBatchInsert;
|
18
18
|
import org.embulk.output.oracle.InsertMethod;
|
19
19
|
import org.embulk.output.oracle.OracleCharset;
|
20
20
|
import org.embulk.output.oracle.OracleOutputConnection;
|
21
21
|
import org.embulk.output.oracle.OracleOutputConnector;
|
22
|
-
import org.embulk.output.oracle.setter.OracleColumnSetterFactory;
|
23
22
|
import org.embulk.spi.PageReader;
|
24
|
-
import org.embulk.spi.time.TimestampFormatter;
|
25
|
-
|
26
|
-
import com.google.common.base.Optional;
|
27
23
|
|
28
24
|
public class OracleOutputPlugin
|
29
25
|
extends AbstractJdbcOutputPlugin
|
30
26
|
{
|
31
|
-
private static final int MAX_TABLE_NAME_LENGTH = 30;
|
32
|
-
|
33
27
|
public interface OraclePluginTask
|
34
28
|
extends PluginTask
|
35
29
|
{
|
@@ -71,6 +65,15 @@ public class OracleOutputPlugin
|
|
71
65
|
return OraclePluginTask.class;
|
72
66
|
}
|
73
67
|
|
68
|
+
@Override
|
69
|
+
protected Features getFeatures(PluginTask task)
|
70
|
+
{
|
71
|
+
return new Features()
|
72
|
+
.setMaxTableNameLength(30)
|
73
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
74
|
+
.setIgnoreMergeKeys(false);
|
75
|
+
}
|
76
|
+
|
74
77
|
@Override
|
75
78
|
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
76
79
|
{
|
@@ -111,18 +114,20 @@ public class OracleOutputPlugin
|
|
111
114
|
}
|
112
115
|
|
113
116
|
Properties props = new Properties();
|
117
|
+
props.putAll(oracleTask.getOptions());
|
118
|
+
|
114
119
|
props.setProperty("user", oracleTask.getUser());
|
120
|
+
logger.info("Connecting to {} options {}", url, props);
|
115
121
|
props.setProperty("password", oracleTask.getPassword());
|
116
|
-
props.putAll(oracleTask.getOptions());
|
117
122
|
|
118
123
|
return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
|
119
124
|
}
|
120
125
|
|
121
126
|
@Override
|
122
|
-
protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
|
127
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
123
128
|
{
|
124
|
-
if (
|
125
|
-
throw new UnsupportedOperationException("
|
129
|
+
if (mergeKeys.isPresent()) {
|
130
|
+
throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
|
126
131
|
}
|
127
132
|
|
128
133
|
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
@@ -131,7 +136,7 @@ public class OracleOutputPlugin
|
|
131
136
|
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
132
137
|
OracleCharset charset;
|
133
138
|
try (JdbcOutputConnection connection = connector.connect(true)) {
|
134
|
-
charset = ((OracleOutputConnection)connection).
|
139
|
+
charset = ((OracleOutputConnection)connection).getOracleCharset();
|
135
140
|
}
|
136
141
|
|
137
142
|
return new DirectBatchInsert(
|
@@ -143,55 +148,6 @@ public class OracleOutputPlugin
|
|
143
148
|
oracleTask.getBatchSize());
|
144
149
|
}
|
145
150
|
|
146
|
-
return new StandardBatchInsert(getConnector(task, true));
|
151
|
+
return new StandardBatchInsert(getConnector(task, true), mergeKeys);
|
147
152
|
}
|
148
|
-
|
149
|
-
@Override
|
150
|
-
protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
|
151
|
-
TimestampFormatter timestampFormatter)
|
152
|
-
{
|
153
|
-
return new OracleColumnSetterFactory(batch, pageReader, timestampFormatter);
|
154
|
-
}
|
155
|
-
|
156
|
-
@Override
|
157
|
-
protected String generateSwapTableName(PluginTask task) throws SQLException
|
158
|
-
{
|
159
|
-
return generateSwapTableName(task, "_bl_tmp", MAX_TABLE_NAME_LENGTH);
|
160
|
-
}
|
161
|
-
|
162
|
-
// TODO move this method to AbstractJdbcOutputPlugin
|
163
|
-
protected String generateSwapTableName(PluginTask task, String suffix, int maxTableNameLength) throws SQLException
|
164
|
-
{
|
165
|
-
String tableName = task.getTable();
|
166
|
-
String uniqueSuffix = getTransactionUniqueName() + suffix;
|
167
|
-
|
168
|
-
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) { // + 1 for '_'
|
169
|
-
// truncate transaction unique name
|
170
|
-
int suffixLength = Math.max(maxTableNameLength - tableName.length() - 1, suffix.length() + 8); // include 8 characters of the transaction name at least
|
171
|
-
uniqueSuffix = uniqueSuffix.substring(uniqueSuffix.length() - suffixLength);
|
172
|
-
}
|
173
|
-
|
174
|
-
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {
|
175
|
-
// use truncated table name
|
176
|
-
int truncLength = maxTableNameLength - uniqueSuffix.length() - 1;
|
177
|
-
while (true) {
|
178
|
-
truncLength--;
|
179
|
-
if (truncLength <= 0) {
|
180
|
-
throw new ConfigException("Table name is too long to generate temporary table name");
|
181
|
-
}
|
182
|
-
tableName = tableName.substring(0, truncLength);
|
183
|
-
//if (!connection.tableExists(tableName)) {
|
184
|
-
// TODO this doesn't help. Rather than truncating more characters,
|
185
|
-
// here needs to replace characters with random characters. But
|
186
|
-
// to make the result deterministic. So, an idea is replacing
|
187
|
-
// the last character to the first (second, third, ... for each loop)
|
188
|
-
// of md5(original table name).
|
189
|
-
return tableName + "_" + uniqueSuffix;
|
190
|
-
//}
|
191
|
-
}
|
192
|
-
}
|
193
|
-
|
194
|
-
return tableName + "_" + uniqueSuffix;
|
195
|
-
}
|
196
|
-
|
197
153
|
}
|