embulk-output-oracle 0.2.4 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +155 -110
- data/build.gradle +6 -6
- data/classpath/embulk-output-jdbc-0.3.0.jar +0 -0
- data/classpath/embulk-output-oracle-0.3.0.jar +0 -0
- data/lib/embulk/output/oracle.rb +3 -3
- data/src/main/cpp/common/dir-path-load.cpp +424 -424
- data/src/main/cpp/common/dir-path-load.h +36 -36
- data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
- data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
- data/src/main/cpp/linux/build.sh +21 -21
- data/src/main/cpp/win/build.bat +31 -31
- data/src/main/cpp/win/dllmain.cpp +25 -25
- data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
- data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
- data/src/main/java/org/embulk/output/OracleOutputPlugin.java +22 -66
- data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +289 -289
- data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
- data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -19
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -134
- data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
- data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
- data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
- data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
- data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
- data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
- data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +99 -99
- data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
- data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
- data/src/test/cpp/linux/build.sh +19 -19
- data/src/test/cpp/win/build.bat +28 -28
- data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
- data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
- data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
- data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
- data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
- data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +120 -120
- data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +100 -100
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +172 -161
- data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +445 -413
- data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
- data/src/test/resources/data/test1/test1.csv +3 -3
- data/src/test/resources/yml/test-insert-direct.yml +26 -26
- data/src/test/resources/yml/test-insert-oci-split.yml +26 -26
- data/src/test/resources/yml/test-insert-oci.yml +26 -26
- data/src/test/resources/yml/test-insert.yml +25 -25
- data/src/test/resources/yml/test-replace-long-name-multibyte.yml +25 -0
- data/src/test/resources/yml/test-replace-long-name.yml +25 -25
- data/src/test/resources/yml/test-replace.yml +25 -25
- data/src/test/resources/yml/test-string-timestamp.yml +25 -0
- data/src/test/resources/yml/test-url.yml +24 -24
- metadata +6 -5
- data/classpath/embulk-output-jdbc-0.2.4.jar +0 -0
- data/classpath/embulk-output-oracle-0.2.4.jar +0 -0
- data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java +0 -31
@@ -1,35 +1,29 @@
|
|
1
1
|
package org.embulk.output;
|
2
2
|
|
3
|
+
import java.util.List;
|
4
|
+
import java.util.Properties;
|
3
5
|
import java.io.IOException;
|
4
|
-
import java.
|
6
|
+
import java.nio.ByteBuffer;
|
5
7
|
import java.sql.SQLException;
|
6
|
-
import
|
7
|
-
|
8
|
+
import com.google.common.base.Optional;
|
9
|
+
import com.google.common.collect.ImmutableSet;
|
8
10
|
import org.embulk.config.Config;
|
9
11
|
import org.embulk.config.ConfigDefault;
|
10
|
-
import org.embulk.config.ConfigException;
|
11
12
|
import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
|
12
13
|
import org.embulk.output.jdbc.BatchInsert;
|
13
14
|
import org.embulk.output.jdbc.JdbcOutputConnection;
|
14
15
|
import org.embulk.output.jdbc.JdbcOutputConnector;
|
15
16
|
import org.embulk.output.jdbc.StandardBatchInsert;
|
16
|
-
import org.embulk.output.jdbc.setter.ColumnSetterFactory;
|
17
17
|
import org.embulk.output.oracle.DirectBatchInsert;
|
18
18
|
import org.embulk.output.oracle.InsertMethod;
|
19
19
|
import org.embulk.output.oracle.OracleCharset;
|
20
20
|
import org.embulk.output.oracle.OracleOutputConnection;
|
21
21
|
import org.embulk.output.oracle.OracleOutputConnector;
|
22
|
-
import org.embulk.output.oracle.setter.OracleColumnSetterFactory;
|
23
22
|
import org.embulk.spi.PageReader;
|
24
|
-
import org.embulk.spi.time.TimestampFormatter;
|
25
|
-
|
26
|
-
import com.google.common.base.Optional;
|
27
23
|
|
28
24
|
public class OracleOutputPlugin
|
29
25
|
extends AbstractJdbcOutputPlugin
|
30
26
|
{
|
31
|
-
private static final int MAX_TABLE_NAME_LENGTH = 30;
|
32
|
-
|
33
27
|
public interface OraclePluginTask
|
34
28
|
extends PluginTask
|
35
29
|
{
|
@@ -71,6 +65,15 @@ public class OracleOutputPlugin
|
|
71
65
|
return OraclePluginTask.class;
|
72
66
|
}
|
73
67
|
|
68
|
+
@Override
|
69
|
+
protected Features getFeatures(PluginTask task)
|
70
|
+
{
|
71
|
+
return new Features()
|
72
|
+
.setMaxTableNameLength(30)
|
73
|
+
.setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
|
74
|
+
.setIgnoreMergeKeys(false);
|
75
|
+
}
|
76
|
+
|
74
77
|
@Override
|
75
78
|
protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
|
76
79
|
{
|
@@ -111,18 +114,20 @@ public class OracleOutputPlugin
|
|
111
114
|
}
|
112
115
|
|
113
116
|
Properties props = new Properties();
|
117
|
+
props.putAll(oracleTask.getOptions());
|
118
|
+
|
114
119
|
props.setProperty("user", oracleTask.getUser());
|
120
|
+
logger.info("Connecting to {} options {}", url, props);
|
115
121
|
props.setProperty("password", oracleTask.getPassword());
|
116
|
-
props.putAll(oracleTask.getOptions());
|
117
122
|
|
118
123
|
return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
|
119
124
|
}
|
120
125
|
|
121
126
|
@Override
|
122
|
-
protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
|
127
|
+
protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
|
123
128
|
{
|
124
|
-
if (
|
125
|
-
throw new UnsupportedOperationException("
|
129
|
+
if (mergeKeys.isPresent()) {
|
130
|
+
throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
|
126
131
|
}
|
127
132
|
|
128
133
|
OraclePluginTask oracleTask = (OraclePluginTask) task;
|
@@ -131,7 +136,7 @@ public class OracleOutputPlugin
|
|
131
136
|
if (oracleTask.getInsertMethod() == InsertMethod.oci) {
|
132
137
|
OracleCharset charset;
|
133
138
|
try (JdbcOutputConnection connection = connector.connect(true)) {
|
134
|
-
charset = ((OracleOutputConnection)connection).
|
139
|
+
charset = ((OracleOutputConnection)connection).getOracleCharset();
|
135
140
|
}
|
136
141
|
|
137
142
|
return new DirectBatchInsert(
|
@@ -143,55 +148,6 @@ public class OracleOutputPlugin
|
|
143
148
|
oracleTask.getBatchSize());
|
144
149
|
}
|
145
150
|
|
146
|
-
return new StandardBatchInsert(getConnector(task, true));
|
151
|
+
return new StandardBatchInsert(getConnector(task, true), mergeKeys);
|
147
152
|
}
|
148
|
-
|
149
|
-
@Override
|
150
|
-
protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
|
151
|
-
TimestampFormatter timestampFormatter)
|
152
|
-
{
|
153
|
-
return new OracleColumnSetterFactory(batch, pageReader, timestampFormatter);
|
154
|
-
}
|
155
|
-
|
156
|
-
@Override
|
157
|
-
protected String generateSwapTableName(PluginTask task) throws SQLException
|
158
|
-
{
|
159
|
-
return generateSwapTableName(task, "_bl_tmp", MAX_TABLE_NAME_LENGTH);
|
160
|
-
}
|
161
|
-
|
162
|
-
// TODO move this method to AbstractJdbcOutputPlugin
|
163
|
-
protected String generateSwapTableName(PluginTask task, String suffix, int maxTableNameLength) throws SQLException
|
164
|
-
{
|
165
|
-
String tableName = task.getTable();
|
166
|
-
String uniqueSuffix = getTransactionUniqueName() + suffix;
|
167
|
-
|
168
|
-
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) { // + 1 for '_'
|
169
|
-
// truncate transaction unique name
|
170
|
-
int suffixLength = Math.max(maxTableNameLength - tableName.length() - 1, suffix.length() + 8); // include 8 characters of the transaction name at least
|
171
|
-
uniqueSuffix = uniqueSuffix.substring(uniqueSuffix.length() - suffixLength);
|
172
|
-
}
|
173
|
-
|
174
|
-
if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {
|
175
|
-
// use truncated table name
|
176
|
-
int truncLength = maxTableNameLength - uniqueSuffix.length() - 1;
|
177
|
-
while (true) {
|
178
|
-
truncLength--;
|
179
|
-
if (truncLength <= 0) {
|
180
|
-
throw new ConfigException("Table name is too long to generate temporary table name");
|
181
|
-
}
|
182
|
-
tableName = tableName.substring(0, truncLength);
|
183
|
-
//if (!connection.tableExists(tableName)) {
|
184
|
-
// TODO this doesn't help. Rather than truncating more characters,
|
185
|
-
// here needs to replace characters with random characters. But
|
186
|
-
// to make the result deterministic. So, an idea is replacing
|
187
|
-
// the last character to the first (second, third, ... for each loop)
|
188
|
-
// of md5(original table name).
|
189
|
-
return tableName + "_" + uniqueSuffix;
|
190
|
-
//}
|
191
|
-
}
|
192
|
-
}
|
193
|
-
|
194
|
-
return tableName + "_" + uniqueSuffix;
|
195
|
-
}
|
196
|
-
|
197
153
|
}
|