embulk-output-oracle 0.2.4 → 0.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +155 -110
  3. data/build.gradle +6 -6
  4. data/classpath/embulk-output-jdbc-0.3.0.jar +0 -0
  5. data/classpath/embulk-output-oracle-0.3.0.jar +0 -0
  6. data/lib/embulk/output/oracle.rb +3 -3
  7. data/src/main/cpp/common/dir-path-load.cpp +424 -424
  8. data/src/main/cpp/common/dir-path-load.h +36 -36
  9. data/src/main/cpp/common/embulk-output-oracle.cpp +196 -196
  10. data/src/main/cpp/common/org_embulk_output_oracle_oci_OCI.h +77 -77
  11. data/src/main/cpp/linux/build.sh +21 -21
  12. data/src/main/cpp/win/build.bat +31 -31
  13. data/src/main/cpp/win/dllmain.cpp +25 -25
  14. data/src/main/cpp/win/embulk-output-oracle.sln +39 -39
  15. data/src/main/cpp/win/embulk-output-oracle.vcxproj +175 -175
  16. data/src/main/java/org/embulk/output/OracleOutputPlugin.java +22 -66
  17. data/src/main/java/org/embulk/output/oracle/DirectBatchInsert.java +289 -289
  18. data/src/main/java/org/embulk/output/oracle/InsertMethod.java +8 -8
  19. data/src/main/java/org/embulk/output/oracle/OracleCharset.java +32 -19
  20. data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java +165 -134
  21. data/src/main/java/org/embulk/output/oracle/OracleOutputConnector.java +49 -49
  22. data/src/main/java/org/embulk/output/oracle/TimestampFormat.java +37 -37
  23. data/src/main/java/org/embulk/output/oracle/oci/ColumnDefinition.java +26 -26
  24. data/src/main/java/org/embulk/output/oracle/oci/OCI.java +139 -139
  25. data/src/main/java/org/embulk/output/oracle/oci/OCIManager.java +64 -64
  26. data/src/main/java/org/embulk/output/oracle/oci/OCIWrapper.java +96 -96
  27. data/src/main/java/org/embulk/output/oracle/oci/RowBuffer.java +99 -99
  28. data/src/main/java/org/embulk/output/oracle/oci/TableDefinition.java +24 -24
  29. data/src/test/cpp/common/embulk-output-oracle-test.cpp +69 -69
  30. data/src/test/cpp/linux/build.sh +19 -19
  31. data/src/test/cpp/win/build.bat +28 -28
  32. data/src/test/cpp/win/embulk-output-oracle-test.vcxproj +154 -154
  33. data/src/test/java/org/embulk/input/filesplit/LocalFileSplitInputPlugin.java +187 -187
  34. data/src/test/java/org/embulk/input/filesplit/PartialFile.java +49 -49
  35. data/src/test/java/org/embulk/input/filesplit/PartialFileInputStream.java +154 -154
  36. data/src/test/java/org/embulk/output/oracle/ChildFirstClassLoader.java +42 -42
  37. data/src/test/java/org/embulk/output/oracle/EmbulkPluginTester.java +120 -120
  38. data/src/test/java/org/embulk/output/oracle/EmptyConfigSource.java +100 -100
  39. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java +172 -161
  40. data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTestImpl.java +445 -413
  41. data/src/test/java/org/embulk/output/oracle/TimestampFormatTest.java +57 -57
  42. data/src/test/resources/data/test1/test1.csv +3 -3
  43. data/src/test/resources/yml/test-insert-direct.yml +26 -26
  44. data/src/test/resources/yml/test-insert-oci-split.yml +26 -26
  45. data/src/test/resources/yml/test-insert-oci.yml +26 -26
  46. data/src/test/resources/yml/test-insert.yml +25 -25
  47. data/src/test/resources/yml/test-replace-long-name-multibyte.yml +25 -0
  48. data/src/test/resources/yml/test-replace-long-name.yml +25 -25
  49. data/src/test/resources/yml/test-replace.yml +25 -25
  50. data/src/test/resources/yml/test-string-timestamp.yml +25 -0
  51. data/src/test/resources/yml/test-url.yml +24 -24
  52. metadata +6 -5
  53. data/classpath/embulk-output-jdbc-0.2.4.jar +0 -0
  54. data/classpath/embulk-output-oracle-0.2.4.jar +0 -0
  55. data/src/main/java/org/embulk/output/oracle/setter/OracleColumnSetterFactory.java +0 -31
@@ -1,35 +1,29 @@
1
1
  package org.embulk.output;
2
2
 
3
+ import java.util.List;
4
+ import java.util.Properties;
3
5
  import java.io.IOException;
4
- import java.lang.UnsupportedOperationException;
6
+ import java.nio.ByteBuffer;
5
7
  import java.sql.SQLException;
6
- import java.util.Properties;
7
-
8
+ import com.google.common.base.Optional;
9
+ import com.google.common.collect.ImmutableSet;
8
10
  import org.embulk.config.Config;
9
11
  import org.embulk.config.ConfigDefault;
10
- import org.embulk.config.ConfigException;
11
12
  import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
12
13
  import org.embulk.output.jdbc.BatchInsert;
13
14
  import org.embulk.output.jdbc.JdbcOutputConnection;
14
15
  import org.embulk.output.jdbc.JdbcOutputConnector;
15
16
  import org.embulk.output.jdbc.StandardBatchInsert;
16
- import org.embulk.output.jdbc.setter.ColumnSetterFactory;
17
17
  import org.embulk.output.oracle.DirectBatchInsert;
18
18
  import org.embulk.output.oracle.InsertMethod;
19
19
  import org.embulk.output.oracle.OracleCharset;
20
20
  import org.embulk.output.oracle.OracleOutputConnection;
21
21
  import org.embulk.output.oracle.OracleOutputConnector;
22
- import org.embulk.output.oracle.setter.OracleColumnSetterFactory;
23
22
  import org.embulk.spi.PageReader;
24
- import org.embulk.spi.time.TimestampFormatter;
25
-
26
- import com.google.common.base.Optional;
27
23
 
28
24
  public class OracleOutputPlugin
29
25
  extends AbstractJdbcOutputPlugin
30
26
  {
31
- private static final int MAX_TABLE_NAME_LENGTH = 30;
32
-
33
27
  public interface OraclePluginTask
34
28
  extends PluginTask
35
29
  {
@@ -71,6 +65,15 @@ public class OracleOutputPlugin
71
65
  return OraclePluginTask.class;
72
66
  }
73
67
 
68
+ @Override
69
+ protected Features getFeatures(PluginTask task)
70
+ {
71
+ return new Features()
72
+ .setMaxTableNameLength(30)
73
+ .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
74
+ .setIgnoreMergeKeys(false);
75
+ }
76
+
74
77
  @Override
75
78
  protected OracleOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
76
79
  {
@@ -111,18 +114,20 @@ public class OracleOutputPlugin
111
114
  }
112
115
 
113
116
  Properties props = new Properties();
117
+ props.putAll(oracleTask.getOptions());
118
+
114
119
  props.setProperty("user", oracleTask.getUser());
120
+ logger.info("Connecting to {} options {}", url, props);
115
121
  props.setProperty("password", oracleTask.getPassword());
116
- props.putAll(oracleTask.getOptions());
117
122
 
118
123
  return new OracleOutputConnector(url, props, oracleTask.getInsertMethod() == InsertMethod.direct);
119
124
  }
120
125
 
121
126
  @Override
122
- protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
127
+ protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
123
128
  {
124
- if (task.getMode().isMerge()) {
125
- throw new UnsupportedOperationException("mode 'merge' is not implemented for this type");
129
+ if (mergeKeys.isPresent()) {
130
+ throw new UnsupportedOperationException("Oracle output plugin doesn't support 'merge_direct' mode.");
126
131
  }
127
132
 
128
133
  OraclePluginTask oracleTask = (OraclePluginTask) task;
@@ -131,7 +136,7 @@ public class OracleOutputPlugin
131
136
  if (oracleTask.getInsertMethod() == InsertMethod.oci) {
132
137
  OracleCharset charset;
133
138
  try (JdbcOutputConnection connection = connector.connect(true)) {
134
- charset = ((OracleOutputConnection)connection).getCharset();
139
+ charset = ((OracleOutputConnection)connection).getOracleCharset();
135
140
  }
136
141
 
137
142
  return new DirectBatchInsert(
@@ -143,55 +148,6 @@ public class OracleOutputPlugin
143
148
  oracleTask.getBatchSize());
144
149
  }
145
150
 
146
- return new StandardBatchInsert(getConnector(task, true));
151
+ return new StandardBatchInsert(getConnector(task, true), mergeKeys);
147
152
  }
148
-
149
- @Override
150
- protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
151
- TimestampFormatter timestampFormatter)
152
- {
153
- return new OracleColumnSetterFactory(batch, pageReader, timestampFormatter);
154
- }
155
-
156
- @Override
157
- protected String generateSwapTableName(PluginTask task) throws SQLException
158
- {
159
- return generateSwapTableName(task, "_bl_tmp", MAX_TABLE_NAME_LENGTH);
160
- }
161
-
162
- // TODO move this method to AbstractJdbcOutputPlugin
163
- protected String generateSwapTableName(PluginTask task, String suffix, int maxTableNameLength) throws SQLException
164
- {
165
- String tableName = task.getTable();
166
- String uniqueSuffix = getTransactionUniqueName() + suffix;
167
-
168
- if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) { // + 1 for '_'
169
- // truncate transaction unique name
170
- int suffixLength = Math.max(maxTableNameLength - tableName.length() - 1, suffix.length() + 8); // include 8 characters of the transaction name at least
171
- uniqueSuffix = uniqueSuffix.substring(uniqueSuffix.length() - suffixLength);
172
- }
173
-
174
- if (tableName.length() + uniqueSuffix.length() + 1 > maxTableNameLength) {
175
- // use truncated table name
176
- int truncLength = maxTableNameLength - uniqueSuffix.length() - 1;
177
- while (true) {
178
- truncLength--;
179
- if (truncLength <= 0) {
180
- throw new ConfigException("Table name is too long to generate temporary table name");
181
- }
182
- tableName = tableName.substring(0, truncLength);
183
- //if (!connection.tableExists(tableName)) {
184
- // TODO this doesn't help. Rather than truncating more characters,
185
- // here needs to replace characters with random characters. But
186
- // to make the result deterministic. So, an idea is replacing
187
- // the last character to the first (second, third, ... for each loop)
188
- // of md5(original table name).
189
- return tableName + "_" + uniqueSuffix;
190
- //}
191
- }
192
- }
193
-
194
- return tableName + "_" + uniqueSuffix;
195
- }
196
-
197
153
  }