embulk-output-teradata 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/.DS_Store +0 -0
  3. data/README.md +0 -8
  4. data/build.gradle +5 -2
  5. data/embulk-output-teradata.iml +12 -0
  6. data/gradle/wrapper/gradle-wrapper.properties +2 -2
  7. data/gradlew +0 -0
  8. data/lib/embulk-output-jdbc-0.7.11.jar +0 -0
  9. data/src/main/java/org/embulk/output/teradata/TeradataOutputPlugin.java +13 -52
  10. data/src/main/java/org/embulk/output/teradata/jdbc/TeradataBatchInsert.java +37 -0
  11. data/src/main/java/org/embulk/output/teradata/jdbc/TeradataOutputConnection.java +307 -0
  12. data/src/main/java/org/embulk/output/teradata/jdbc/TeradataOutputConnector.java +46 -0
  13. metadata +10 -40
  14. data/src/main/java/org/embulk/output/teradata/JdbcOutputPlugin.java +0 -134
  15. data/src/main/java/org/embulk/output/teradata/TeradataOutputConnection.java +0 -19
  16. data/src/main/java/org/embulk/output/teradata/jdbc/AbstractJdbcOutputPlugin.java +0 -1144
  17. data/src/main/java/org/embulk/output/teradata/jdbc/BatchInsert.java +0 -52
  18. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcColumn.java +0 -134
  19. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcColumnOption.java +0 -34
  20. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcOutputConnection.java +0 -549
  21. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcOutputConnector.java +0 -8
  22. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcSchema.java +0 -79
  23. data/src/main/java/org/embulk/output/teradata/jdbc/JdbcUtils.java +0 -153
  24. data/src/main/java/org/embulk/output/teradata/jdbc/MergeConfig.java +0 -23
  25. data/src/main/java/org/embulk/output/teradata/jdbc/StandardBatchInsert.java +0 -201
  26. data/src/main/java/org/embulk/output/teradata/jdbc/TimestampFormat.java +0 -37
  27. data/src/main/java/org/embulk/output/teradata/jdbc/ToString.java +0 -54
  28. data/src/main/java/org/embulk/output/teradata/jdbc/ToStringMap.java +0 -34
  29. data/src/main/java/org/embulk/output/teradata/jdbc/setter/BigDecimalColumnSetter.java +0 -76
  30. data/src/main/java/org/embulk/output/teradata/jdbc/setter/BooleanColumnSetter.java +0 -61
  31. data/src/main/java/org/embulk/output/teradata/jdbc/setter/ByteColumnSetter.java +0 -84
  32. data/src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetter.java +0 -48
  33. data/src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetterFactory.java +0 -199
  34. data/src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetterVisitor.java +0 -110
  35. data/src/main/java/org/embulk/output/teradata/jdbc/setter/DefaultValueSetter.java +0 -50
  36. data/src/main/java/org/embulk/output/teradata/jdbc/setter/DoubleColumnSetter.java +0 -68
  37. data/src/main/java/org/embulk/output/teradata/jdbc/setter/FloatColumnSetter.java +0 -68
  38. data/src/main/java/org/embulk/output/teradata/jdbc/setter/IntColumnSetter.java +0 -84
  39. data/src/main/java/org/embulk/output/teradata/jdbc/setter/JsonColumnSetter.java +0 -61
  40. data/src/main/java/org/embulk/output/teradata/jdbc/setter/LongColumnSetter.java +0 -80
  41. data/src/main/java/org/embulk/output/teradata/jdbc/setter/NStringColumnSetter.java +0 -66
  42. data/src/main/java/org/embulk/output/teradata/jdbc/setter/NullColumnSetter.java +0 -61
  43. data/src/main/java/org/embulk/output/teradata/jdbc/setter/NullDefaultValueSetter.java +0 -111
  44. data/src/main/java/org/embulk/output/teradata/jdbc/setter/PassThroughColumnSetter.java +0 -66
  45. data/src/main/java/org/embulk/output/teradata/jdbc/setter/ShortColumnSetter.java +0 -84
  46. data/src/main/java/org/embulk/output/teradata/jdbc/setter/SkipColumnSetter.java +0 -49
  47. data/src/main/java/org/embulk/output/teradata/jdbc/setter/SqlDateColumnSetter.java +0 -66
  48. data/src/main/java/org/embulk/output/teradata/jdbc/setter/SqlTimeColumnSetter.java +0 -66
  49. data/src/main/java/org/embulk/output/teradata/jdbc/setter/SqlTimestampColumnSetter.java +0 -66
  50. data/src/main/java/org/embulk/output/teradata/jdbc/setter/StringColumnSetter.java +0 -66
@@ -0,0 +1,46 @@
1
+ package org.embulk.output.teradata.jdbc;
2
+
3
+ import org.embulk.output.jdbc.JdbcOutputConnector;
4
+
5
+ import java.util.Properties;
6
+ import java.sql.Driver;
7
+ import java.sql.Connection;
8
+ import java.sql.SQLException;
9
+
10
+ public class TeradataOutputConnector
11
+ implements JdbcOutputConnector
12
+ {
13
+ private final Driver driver;
14
+ private final String url;
15
+ private final Properties properties;
16
+
17
+ public TeradataOutputConnector(String url, Properties properties)
18
+ {
19
+ try
20
+ {
21
+ this.driver = new com.teradata.jdbc.TeraDriver();
22
+ }
23
+ catch (Exception ex)
24
+ {
25
+ throw new RuntimeException(ex);
26
+ }
27
+ this.url = url;
28
+ this.properties = properties;
29
+ }
30
+
31
+ @Override
32
+ public TeradataOutputConnection connect(boolean autoCommit) throws SQLException
33
+ {
34
+ Connection c = driver.connect(url, properties);
35
+
36
+ try {
37
+ TeradataOutputConnection con = new TeradataOutputConnection(c, autoCommit);
38
+ c = null;
39
+ return con;
40
+ } finally {
41
+ if (c != null) {
42
+ c.close();
43
+ }
44
+ }
45
+ }
46
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-teradata
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - ebyhr
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-07-15 00:00:00.000000000 Z
11
+ date: 2017-08-30 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -45,6 +45,7 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
+ - .DS_Store
48
49
  - .gitattributes
49
50
  - .gitignore
50
51
  - LICENSE.txt
@@ -52,53 +53,22 @@ files:
52
53
  - build.gradle
53
54
  - config/checkstyle/checkstyle.xml
54
55
  - config/checkstyle/default.xml
56
+ - embulk-output-teradata.iml
55
57
  - gradle/wrapper/gradle-wrapper.jar
56
58
  - gradle/wrapper/gradle-wrapper.properties
57
59
  - gradlew
58
60
  - gradlew.bat
61
+ - lib/embulk-output-jdbc-0.7.11.jar
59
62
  - lib/embulk/output/teradata.rb
60
63
  - lib/tdgssconfig.jar
61
64
  - lib/terajdbc4.jar
62
- - src/main/java/org/embulk/output/teradata/JdbcOutputPlugin.java
63
- - src/main/java/org/embulk/output/teradata/TeradataOutputConnection.java
64
65
  - src/main/java/org/embulk/output/teradata/TeradataOutputPlugin.java
65
- - src/main/java/org/embulk/output/teradata/jdbc/AbstractJdbcOutputPlugin.java
66
- - src/main/java/org/embulk/output/teradata/jdbc/BatchInsert.java
67
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcColumn.java
68
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcColumnOption.java
69
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcOutputConnection.java
70
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcOutputConnector.java
71
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcSchema.java
72
- - src/main/java/org/embulk/output/teradata/jdbc/JdbcUtils.java
73
- - src/main/java/org/embulk/output/teradata/jdbc/MergeConfig.java
74
- - src/main/java/org/embulk/output/teradata/jdbc/StandardBatchInsert.java
75
- - src/main/java/org/embulk/output/teradata/jdbc/TimestampFormat.java
76
- - src/main/java/org/embulk/output/teradata/jdbc/ToString.java
77
- - src/main/java/org/embulk/output/teradata/jdbc/ToStringMap.java
78
- - src/main/java/org/embulk/output/teradata/jdbc/setter/BigDecimalColumnSetter.java
79
- - src/main/java/org/embulk/output/teradata/jdbc/setter/BooleanColumnSetter.java
80
- - src/main/java/org/embulk/output/teradata/jdbc/setter/ByteColumnSetter.java
81
- - src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetter.java
82
- - src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetterFactory.java
83
- - src/main/java/org/embulk/output/teradata/jdbc/setter/ColumnSetterVisitor.java
84
- - src/main/java/org/embulk/output/teradata/jdbc/setter/DefaultValueSetter.java
85
- - src/main/java/org/embulk/output/teradata/jdbc/setter/DoubleColumnSetter.java
86
- - src/main/java/org/embulk/output/teradata/jdbc/setter/FloatColumnSetter.java
87
- - src/main/java/org/embulk/output/teradata/jdbc/setter/IntColumnSetter.java
88
- - src/main/java/org/embulk/output/teradata/jdbc/setter/JsonColumnSetter.java
89
- - src/main/java/org/embulk/output/teradata/jdbc/setter/LongColumnSetter.java
90
- - src/main/java/org/embulk/output/teradata/jdbc/setter/NStringColumnSetter.java
91
- - src/main/java/org/embulk/output/teradata/jdbc/setter/NullColumnSetter.java
92
- - src/main/java/org/embulk/output/teradata/jdbc/setter/NullDefaultValueSetter.java
93
- - src/main/java/org/embulk/output/teradata/jdbc/setter/PassThroughColumnSetter.java
94
- - src/main/java/org/embulk/output/teradata/jdbc/setter/ShortColumnSetter.java
95
- - src/main/java/org/embulk/output/teradata/jdbc/setter/SkipColumnSetter.java
96
- - src/main/java/org/embulk/output/teradata/jdbc/setter/SqlDateColumnSetter.java
97
- - src/main/java/org/embulk/output/teradata/jdbc/setter/SqlTimeColumnSetter.java
98
- - src/main/java/org/embulk/output/teradata/jdbc/setter/SqlTimestampColumnSetter.java
99
- - src/main/java/org/embulk/output/teradata/jdbc/setter/StringColumnSetter.java
66
+ - src/main/java/org/embulk/output/teradata/jdbc/TeradataBatchInsert.java
67
+ - src/main/java/org/embulk/output/teradata/jdbc/TeradataOutputConnection.java
68
+ - src/main/java/org/embulk/output/teradata/jdbc/TeradataOutputConnector.java
100
69
  - src/test/java/org/embulk/output/teradata/TestTeradataOutputPlugin.java
101
- - classpath/embulk-output-teradata-0.1.2.jar
70
+ - classpath/embulk-output-jdbc-0.7.11.jar
71
+ - classpath/embulk-output-teradata-0.1.3.jar
102
72
  - classpath/tdgssconfig.jar
103
73
  - classpath/terajdbc4.jar
104
74
  homepage: https://github.com/ebyhr/embulk-output-teradata
@@ -1,134 +0,0 @@
1
- package org.embulk.output;
2
-
3
- import java.util.Properties;
4
- import java.sql.Driver;
5
- import java.io.IOException;
6
- import java.sql.Connection;
7
- import java.sql.SQLException;
8
-
9
- import com.google.common.base.Optional;
10
- import com.google.common.base.Throwables;
11
- import com.google.common.collect.ImmutableSet;
12
-
13
- import org.embulk.config.Config;
14
- import org.embulk.config.ConfigDefault;
15
- import org.embulk.output.jdbc.*;
16
-
17
- public class JdbcOutputPlugin
18
- extends AbstractJdbcOutputPlugin
19
- {
20
- public interface GenericPluginTask extends PluginTask
21
- {
22
- @Config("driver_path")
23
- @ConfigDefault("null")
24
- public Optional<String> getDriverPath();
25
-
26
- @Config("driver_class")
27
- public String getDriverClass();
28
-
29
- @Config("url")
30
- public String getUrl();
31
-
32
- @Config("user")
33
- @ConfigDefault("null")
34
- public Optional<String> getUser();
35
-
36
- @Config("password")
37
- @ConfigDefault("null")
38
- public Optional<String> getPassword();
39
-
40
- @Config("schema")
41
- @ConfigDefault("null")
42
- public Optional<String> getSchema();
43
-
44
- @Config("max_table_name_length")
45
- @ConfigDefault("30")
46
- public int getMaxTableNameLength();
47
- }
48
-
49
- @Override
50
- protected Class<? extends PluginTask> getTaskClass()
51
- {
52
- return GenericPluginTask.class;
53
- }
54
-
55
- @Override
56
- protected Features getFeatures(PluginTask task)
57
- {
58
- GenericPluginTask t = (GenericPluginTask) task;
59
- return new Features()
60
- .setMaxTableNameLength(t.getMaxTableNameLength())
61
- .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE));
62
- }
63
-
64
- @Override
65
- protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
66
- {
67
- GenericPluginTask t = (GenericPluginTask) task;
68
-
69
- if (t.getDriverPath().isPresent()) {
70
- loadDriverJar(t.getDriverPath().get());
71
- }
72
-
73
- Properties props = new Properties();
74
-
75
- props.putAll(t.getOptions());
76
-
77
- if (t.getUser().isPresent()) {
78
- props.setProperty("user", t.getUser().get());
79
- }
80
- logger.info("Connecting to {} options {}", t.getUrl(), props);
81
- if (t.getPassword().isPresent()) {
82
- props.setProperty("password", t.getPassword().get());
83
- }
84
-
85
- return new GenericOutputConnector(t.getUrl(), props, t.getDriverClass(),
86
- t.getSchema().orNull());
87
- }
88
-
89
- private static class GenericOutputConnector
90
- implements JdbcOutputConnector
91
- {
92
- private final Driver driver;
93
- private final String url;
94
- private final Properties properties;
95
- private final String schemaName;
96
-
97
- public GenericOutputConnector(String url, Properties properties, String driverClass,
98
- String schemaName)
99
- {
100
- try {
101
- // TODO check Class.forName(driverClass) is a Driver before newInstance
102
- // for security
103
- this.driver = (Driver) Class.forName(driverClass).newInstance();
104
- } catch (Exception ex) {
105
- throw Throwables.propagate(ex);
106
- }
107
- this.url = url;
108
- this.properties = properties;
109
- this.schemaName = schemaName;
110
- }
111
-
112
- @Override
113
- public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
114
- {
115
- Connection c = driver.connect(url, properties);
116
- try {
117
- c.setAutoCommit(autoCommit);
118
- JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
119
- c = null;
120
- return con;
121
- } finally {
122
- if (c != null) {
123
- c.close();
124
- }
125
- }
126
- }
127
- }
128
-
129
- @Override
130
- protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
131
- {
132
- return new StandardBatchInsert(getConnector(task, true), mergeConfig);
133
- }
134
- }
@@ -1,19 +0,0 @@
1
- package org.embulk.output.teradata;
2
-
3
- import java.sql.Connection;
4
- import java.sql.SQLException;
5
- import org.slf4j.Logger;
6
- import org.embulk.spi.Exec;
7
- import org.embulk.output.jdbc.JdbcOutputConnection;
8
-
9
- public class TeradataOutputConnection
10
- extends JdbcOutputConnection
11
- {
12
- private final Logger logger = Exec.getLogger(TeradataOutputConnection.class);
13
-
14
- public TeradataOutputConnection(Connection connection, String schemaName)
15
- throws SQLException
16
- {
17
- super(connection, schemaName);
18
- }
19
- }
@@ -1,1144 +0,0 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.util.HashSet;
4
- import java.util.List;
5
- import java.util.Map;
6
- import java.util.Locale;
7
- import java.util.Set;
8
- import java.util.concurrent.ExecutionException;
9
- import java.io.IOException;
10
- import java.nio.charset.Charset;
11
- import java.nio.file.Paths;
12
- import java.sql.Types;
13
- import java.sql.ResultSet;
14
- import java.sql.DatabaseMetaData;
15
- import java.sql.SQLException;
16
-
17
- import org.embulk.spi.util.RetryExecutor;
18
- import org.slf4j.Logger;
19
- import org.joda.time.DateTimeZone;
20
-
21
- import com.fasterxml.jackson.annotation.JsonCreator;
22
- import com.fasterxml.jackson.annotation.JsonValue;
23
- import com.fasterxml.jackson.annotation.JsonProperty;
24
- import com.google.common.base.Optional;
25
- import com.google.common.base.Function;
26
- import com.google.common.base.Supplier;
27
- import com.google.common.base.Throwables;
28
- import com.google.common.collect.Lists;
29
- import com.google.common.collect.ImmutableList;
30
- import com.google.common.collect.ImmutableSet;
31
-
32
- import org.embulk.config.Config;
33
- import org.embulk.config.ConfigDefault;
34
- import org.embulk.config.ConfigDiff;
35
- import org.embulk.config.ConfigException;
36
- import org.embulk.config.ConfigSource;
37
- import org.embulk.config.Task;
38
- import org.embulk.config.TaskReport;
39
- import org.embulk.config.TaskSource;
40
- import org.embulk.plugin.PluginClassLoader;
41
- import org.embulk.spi.Exec;
42
- import org.embulk.spi.Column;
43
- import org.embulk.spi.ColumnVisitor;
44
- import org.embulk.spi.OutputPlugin;
45
- import org.embulk.spi.Schema;
46
- import org.embulk.spi.TransactionalPageOutput;
47
- import org.embulk.spi.Page;
48
- import org.embulk.spi.PageReader;
49
- import org.embulk.spi.time.Timestamp;
50
- import org.embulk.output.jdbc.setter.ColumnSetter;
51
- import org.embulk.output.jdbc.setter.ColumnSetterFactory;
52
- import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
53
- import org.embulk.spi.util.RetryExecutor.Retryable;
54
-
55
- import static org.embulk.spi.util.RetryExecutor.retryExecutor;
56
- import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
57
-
58
- public abstract class AbstractJdbcOutputPlugin
59
- implements OutputPlugin
60
- {
61
- private final static Set<String> loadedJarGlobs = new HashSet<String>();
62
-
63
- protected final Logger logger = Exec.getLogger(getClass());
64
-
65
- public interface PluginTask
66
- extends Task
67
- {
68
- @Config("options")
69
- @ConfigDefault("{}")
70
- public ToStringMap getOptions();
71
-
72
- @Config("table")
73
- public String getTable();
74
-
75
- @Config("mode")
76
- public Mode getMode();
77
-
78
- @Config("batch_size")
79
- @ConfigDefault("16777216")
80
- // TODO set minimum number
81
- public int getBatchSize();
82
-
83
- @Config("merge_keys")
84
- @ConfigDefault("null")
85
- public Optional<List<String>> getMergeKeys();
86
-
87
- @Config("column_options")
88
- @ConfigDefault("{}")
89
- public Map<String, JdbcColumnOption> getColumnOptions();
90
-
91
- @Config("default_timezone")
92
- @ConfigDefault("\"UTC\"")
93
- public DateTimeZone getDefaultTimeZone();
94
-
95
- @Config("retry_limit")
96
- @ConfigDefault("12")
97
- public int getRetryLimit();
98
-
99
- @Config("retry_wait")
100
- @ConfigDefault("1000")
101
- public int getRetryWait();
102
-
103
- @Config("max_retry_wait")
104
- @ConfigDefault("1800000") // 30 * 60 * 1000
105
- public int getMaxRetryWait();
106
-
107
- @Config("merge_rule")
108
- @ConfigDefault("null")
109
- public Optional<List<String>> getMergeRule();
110
-
111
- @Config("after_load")
112
- @ConfigDefault("null")
113
- public Optional<String> getAfterLoad();
114
-
115
- public void setActualTable(String actualTable);
116
- public String getActualTable();
117
-
118
- public void setMergeKeys(Optional<List<String>> keys);
119
-
120
- public void setFeatures(Features features);
121
- public Features getFeatures();
122
-
123
- public Optional<JdbcSchema> getNewTableSchema();
124
- public void setNewTableSchema(Optional<JdbcSchema> schema);
125
-
126
- public JdbcSchema getTargetTableSchema();
127
- public void setTargetTableSchema(JdbcSchema schema);
128
-
129
- public Optional<List<String>> getIntermediateTables();
130
- public void setIntermediateTables(Optional<List<String>> names);
131
- }
132
-
133
- public static enum LengthSemantics
134
- {
135
- BYTES {
136
- @Override
137
- public int countLength(Charset charset, String s)
138
- {
139
- return charset.encode(s).remaining();
140
- }
141
- },
142
- CHARACTERS {
143
- @Override
144
- public int countLength(Charset charset, String s)
145
- {
146
- return s.length();
147
- }
148
- };
149
-
150
- public abstract int countLength(Charset charset, String s);
151
- }
152
-
153
- public static class Features
154
- {
155
- private int maxTableNameLength = 64;
156
- private LengthSemantics tableNameLengthSemantics = LengthSemantics.BYTES;
157
- private Set<Mode> supportedModes = ImmutableSet.copyOf(Mode.values());
158
- private boolean ignoreMergeKeys = false;
159
-
160
- public Features()
161
- { }
162
-
163
- @JsonProperty
164
- public int getMaxTableNameLength()
165
- {
166
- return maxTableNameLength;
167
- }
168
-
169
- @JsonProperty
170
- public Features setMaxTableNameLength(int bytes)
171
- {
172
- this.maxTableNameLength = bytes;
173
- return this;
174
- }
175
-
176
- public LengthSemantics getTableNameLengthSemantics()
177
- {
178
- return tableNameLengthSemantics;
179
- }
180
-
181
- @JsonProperty
182
- public Features setTableNameLengthSemantics(LengthSemantics tableNameLengthSemantics)
183
- {
184
- this.tableNameLengthSemantics = tableNameLengthSemantics;
185
- return this;
186
- }
187
-
188
- @JsonProperty
189
- public Set<Mode> getSupportedModes()
190
- {
191
- return supportedModes;
192
- }
193
-
194
- @JsonProperty
195
- public Features setSupportedModes(Set<Mode> modes)
196
- {
197
- this.supportedModes = modes;
198
- return this;
199
- }
200
-
201
- @JsonProperty
202
- public boolean getIgnoreMergeKeys()
203
- {
204
- return ignoreMergeKeys;
205
- }
206
-
207
- @JsonProperty
208
- public Features setIgnoreMergeKeys(boolean value)
209
- {
210
- this.ignoreMergeKeys = value;
211
- return this;
212
- }
213
- }
214
-
215
- protected void loadDriverJar(String glob)
216
- {
217
- synchronized (loadedJarGlobs) {
218
- if (!loadedJarGlobs.contains(glob)) {
219
- // TODO match glob
220
- PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
221
- loader.addPath(Paths.get(glob));
222
- loadedJarGlobs.add(glob);
223
- }
224
- }
225
- }
226
-
227
- // for subclasses to add @Config
228
- protected Class<? extends PluginTask> getTaskClass()
229
- {
230
- return PluginTask.class;
231
- }
232
-
233
- protected abstract Features getFeatures(PluginTask task);
234
-
235
- protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
236
-
237
- protected abstract BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException;
238
-
239
- protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
240
- boolean autoCommit) throws SQLException
241
- {
242
- return getConnector(task, retryableMetadataOperation).connect(autoCommit);
243
- }
244
-
245
- public enum Mode {
246
- INSERT,
247
- INSERT_DIRECT,
248
- MERGE,
249
- MERGE_DIRECT,
250
- TRUNCATE_INSERT,
251
- REPLACE;
252
-
253
- @JsonValue
254
- @Override
255
- public String toString()
256
- {
257
- return name().toLowerCase(Locale.ENGLISH);
258
- }
259
-
260
- @JsonCreator
261
- public static Mode fromString(String value)
262
- {
263
- switch(value) {
264
- case "insert":
265
- return INSERT;
266
- case "insert_direct":
267
- return INSERT_DIRECT;
268
- case "merge":
269
- return MERGE;
270
- case "merge_direct":
271
- return MERGE_DIRECT;
272
- case "truncate_insert":
273
- return TRUNCATE_INSERT;
274
- case "replace":
275
- return REPLACE;
276
- default:
277
- throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are insert, insert_direct, merge, merge_direct, truncate_insert, replace", value));
278
- }
279
- }
280
-
281
- /**
282
- * True if this mode directly modifies the target table without creating intermediate tables.
283
- */
284
- public boolean isDirectModify()
285
- {
286
- return this == INSERT_DIRECT || this == MERGE_DIRECT;
287
- }
288
-
289
- /**
290
- * True if this mode merges records on unique keys
291
- */
292
- public boolean isMerge()
293
- {
294
- return this == MERGE || this == MERGE_DIRECT;
295
- }
296
-
297
- /**
298
- * True if this mode creates intermediate table for each tasks.
299
- */
300
- public boolean tempTablePerTask()
301
- {
302
- return this == INSERT || this == MERGE || this == TRUNCATE_INSERT /*this == REPLACE_VIEW*/;
303
- }
304
-
305
- /**
306
- * True if this mode truncates the target table before committing intermediate tables
307
- */
308
- public boolean truncateBeforeCommit()
309
- {
310
- return this == TRUNCATE_INSERT;
311
- }
312
-
313
- /**
314
- * True if this mode uses MERGE statement to commit intermediate tables to the target table
315
- */
316
- public boolean commitByMerge()
317
- {
318
- return this == MERGE;
319
- }
320
-
321
- /**
322
- * True if this mode overwrites schema of the target tables
323
- */
324
- public boolean ignoreTargetTableSchema()
325
- {
326
- return this == REPLACE /*|| this == REPLACE_VIEW*/;
327
- }
328
-
329
- /**
330
- * True if this mode swaps the target tables with intermediate tables to commit
331
- */
332
- public boolean commitBySwapTable()
333
- {
334
- return this == REPLACE;
335
- }
336
- }
337
-
338
- public ConfigDiff transaction(ConfigSource config,
339
- Schema schema, int taskCount,
340
- OutputPlugin.Control control)
341
- {
342
- PluginTask task = config.loadConfig(getTaskClass());
343
- Features features = getFeatures(task);
344
- task.setFeatures(features);
345
-
346
- if (!features.getSupportedModes().contains(task.getMode())) {
347
- throw new ConfigException(String.format("This output type doesn't support '%s'. Supported modes are: %s", task.getMode(), features.getSupportedModes()));
348
- }
349
-
350
- task = begin(task, schema, taskCount);
351
- control.run(task.dump());
352
- return commit(task, schema, taskCount);
353
- }
354
-
355
- public ConfigDiff resume(TaskSource taskSource,
356
- Schema schema, int taskCount,
357
- OutputPlugin.Control control)
358
- {
359
- PluginTask task = taskSource.loadTask(getTaskClass());
360
-
361
- if (!task.getMode().tempTablePerTask()) {
362
- throw new UnsupportedOperationException("inplace mode is not resumable. You need to delete partially-loaded records from the database and restart the entire transaction.");
363
- }
364
-
365
- task = begin(task, schema, taskCount);
366
- control.run(task.dump());
367
- return commit(task, schema, taskCount);
368
- }
369
-
370
- protected String getTransactionUniqueName()
371
- {
372
- // TODO use uuid?
373
- Timestamp t = Exec.session().getTransactionTime();
374
- return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
375
- }
376
-
377
- private PluginTask begin(final PluginTask task,
378
- final Schema schema, final int taskCount)
379
- {
380
- try {
381
- withRetry(task, new IdempotentSqlRunnable() { // no intermediate data if isDirectModify == true
382
- public void run() throws SQLException
383
- {
384
- JdbcOutputConnection con = newConnection(task, true, false);
385
- try {
386
- doBegin(con, task, schema, taskCount);
387
- } finally {
388
- con.close();
389
- }
390
- }
391
- });
392
- } catch (SQLException | InterruptedException ex) {
393
- throw new RuntimeException(ex);
394
- }
395
- return task;
396
- }
397
-
398
- private ConfigDiff commit(final PluginTask task,
399
- Schema schema, final int taskCount)
400
- {
401
- if (!task.getMode().isDirectModify() || task.getAfterLoad().isPresent()) { // no intermediate data if isDirectModify == true
402
- try {
403
- withRetry(task, new IdempotentSqlRunnable() {
404
- public void run() throws SQLException
405
- {
406
- JdbcOutputConnection con = newConnection(task, false, false);
407
- try {
408
- doCommit(con, task, taskCount);
409
- } finally {
410
- con.close();
411
- }
412
- }
413
- });
414
- } catch (SQLException | InterruptedException ex) {
415
- throw new RuntimeException(ex);
416
- }
417
- }
418
- return Exec.newConfigDiff();
419
- }
420
-
421
- public void cleanup(TaskSource taskSource,
422
- Schema schema, final int taskCount,
423
- final List<TaskReport> successTaskReports)
424
- {
425
- final PluginTask task = taskSource.loadTask(getTaskClass());
426
-
427
- if (!task.getMode().isDirectModify()) { // no intermediate data if isDirectModify == true
428
- try {
429
- withRetry(task, new IdempotentSqlRunnable() {
430
- public void run() throws SQLException
431
- {
432
- JdbcOutputConnection con = newConnection(task, true, true);
433
- try {
434
- doCleanup(con, task, taskCount, successTaskReports);
435
- } finally {
436
- con.close();
437
- }
438
- }
439
- });
440
- } catch (SQLException | InterruptedException ex) {
441
- throw new RuntimeException(ex);
442
- }
443
- }
444
- }
445
-
446
- protected void doBegin(JdbcOutputConnection con,
447
- PluginTask task, final Schema schema, int taskCount) throws SQLException
448
- {
449
- if (schema.getColumnCount() == 0) {
450
- throw new ConfigException("No column.");
451
- }
452
-
453
- Mode mode = task.getMode();
454
- logger.info("Using {} mode", mode);
455
-
456
- if (con.tableExists(task.getTable())) {
457
- task.setActualTable(task.getTable());
458
- } else {
459
- String upperTable = task.getTable().toUpperCase();
460
- String lowerTable = task.getTable().toLowerCase();
461
- if (con.tableExists(upperTable)) {
462
- if (con.tableExists(lowerTable)) {
463
- throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
464
- task.getTable(), upperTable, lowerTable));
465
- } else {
466
- task.setActualTable(upperTable);
467
- }
468
- } else {
469
- if (con.tableExists(lowerTable)) {
470
- task.setActualTable(lowerTable);
471
- } else {
472
- task.setActualTable(task.getTable());
473
- }
474
- }
475
- }
476
-
477
- Optional<JdbcSchema> initialTargetTableSchema =
478
- mode.ignoreTargetTableSchema() ?
479
- Optional.<JdbcSchema>absent() :
480
- newJdbcSchemaFromTableIfExists(con, task.getActualTable());
481
-
482
- // TODO get CREATE TABLE statement from task if set
483
- JdbcSchema newTableSchema = applyColumnOptionsToNewTableSchema(
484
- initialTargetTableSchema.or(new Supplier<JdbcSchema>() {
485
- public JdbcSchema get()
486
- {
487
- return newJdbcSchemaForNewTable(schema);
488
- }
489
- }),
490
- task.getColumnOptions());
491
-
492
- // create intermediate tables
493
- if (!mode.isDirectModify()) {
494
- // direct modify mode doesn't need intermediate tables.
495
- ImmutableList.Builder<String> intermTableNames = ImmutableList.builder();
496
- if (mode.tempTablePerTask()) {
497
- String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable(), con, 3,
498
- task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
499
- for (int i = 0; i < taskCount; i++) {
500
- intermTableNames.add(namePrefix + String.format("%03d", i));
501
- }
502
- } else {
503
- String name = generateIntermediateTableNamePrefix(task.getActualTable(), con, 0,
504
- task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
505
- intermTableNames.add(name);
506
- }
507
- // create the intermediate tables here
508
- task.setIntermediateTables(Optional.<List<String>>of(intermTableNames.build()));
509
- for (String name : task.getIntermediateTables().get()) {
510
- // DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
511
- con.dropTableIfExists(name);
512
- // CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
513
- con.createTableIfNotExists(name, newTableSchema);
514
- }
515
- } else {
516
- task.setIntermediateTables(Optional.<List<String>>absent());
517
- }
518
-
519
- // build JdbcSchema from a table
520
- JdbcSchema targetTableSchema;
521
- if (initialTargetTableSchema.isPresent()) {
522
- targetTableSchema = initialTargetTableSchema.get();
523
- task.setNewTableSchema(Optional.<JdbcSchema>absent());
524
- } else if (task.getIntermediateTables().isPresent() && !task.getIntermediateTables().get().isEmpty()) {
525
- String firstItermTable = task.getIntermediateTables().get().get(0);
526
- logger.info(firstItermTable);
527
- targetTableSchema = newJdbcSchemaFromTableIfExists(con, firstItermTable).get();
528
- task.setNewTableSchema(Optional.of(newTableSchema));
529
- } else {
530
- // also create the target table if not exists
531
- // CREATE TABLE IF NOT EXISTS xyz
532
- con.createTableIfNotExists(task.getActualTable(), newTableSchema);
533
- targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getActualTable()).get();
534
- task.setNewTableSchema(Optional.<JdbcSchema>absent());
535
- }
536
- task.setTargetTableSchema(matchSchemaByColumnNames(schema, targetTableSchema));
537
-
538
- // validate column_options
539
- newColumnSetters(
540
- newColumnSetterFactory(null, task.getDefaultTimeZone()), // TODO create a dummy BatchInsert
541
- task.getTargetTableSchema(), schema,
542
- task.getColumnOptions());
543
-
544
- // normalize merge_key parameter for merge modes
545
- if (mode.isMerge()) {
546
- Optional<List<String>> mergeKeys = task.getMergeKeys();
547
- if (task.getFeatures().getIgnoreMergeKeys()) {
548
- if (mergeKeys.isPresent()) {
549
- throw new ConfigException("This output type does not accept 'merge_key' option.");
550
- }
551
- task.setMergeKeys(Optional.<List<String>>of(ImmutableList.<String>of()));
552
- } else if (mergeKeys.isPresent()) {
553
- if (task.getMergeKeys().get().isEmpty()) {
554
- throw new ConfigException("Empty 'merge_keys' option is invalid.");
555
- }
556
- for (String key : mergeKeys.get()) {
557
- if (!targetTableSchema.findColumn(key).isPresent()) {
558
- throw new ConfigException(String.format("Merge key '%s' does not exist in the target table.", key));
559
- }
560
- }
561
- } else {
562
- ImmutableList.Builder<String> builder = ImmutableList.builder();
563
- for (JdbcColumn column : targetTableSchema.getColumns()) {
564
- if (column.isUniqueKey()) {
565
- builder.add(column.getName());
566
- }
567
- }
568
- task.setMergeKeys(Optional.<List<String>>of(builder.build()));
569
- if (task.getMergeKeys().get().isEmpty()) {
570
- throw new ConfigException("Merging mode is used but the target table does not have primary keys. Please set merge_keys option.");
571
- }
572
- }
573
- logger.info("Using merge keys: {}", task.getMergeKeys().get());
574
- } else {
575
- task.setMergeKeys(Optional.<List<String>>absent());
576
- }
577
- }
578
-
579
- protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, DateTimeZone defaultTimeZone)
580
- {
581
- return new ColumnSetterFactory(batch, defaultTimeZone);
582
- }
583
-
584
- protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con,
585
- int suffixLength, int maxLength, LengthSemantics lengthSemantics) throws SQLException
586
- {
587
- Charset tableNameCharset = con.getTableNameCharset();
588
- String tableName = baseTableName;
589
- String suffix = "_bl_tmp";
590
- String uniqueSuffix = getTransactionUniqueName() + suffix;
591
-
592
- // way to count length of table name varies by DBMSs (bytes or characters),
593
- // so truncate swap table name by one character.
594
- while (!checkTableNameLength(tableName + "_" + uniqueSuffix, tableNameCharset, suffixLength, maxLength, lengthSemantics)) {
595
- if (uniqueSuffix.length() > 8 + suffix.length()) {
596
- // truncate transaction unique name
597
- // (include 8 characters of the transaction name at least)
598
- uniqueSuffix = uniqueSuffix.substring(1);
599
- } else {
600
- if (tableName.isEmpty()) {
601
- throw new ConfigException("Table name is too long to generate temporary table name");
602
- }
603
- // truncate table name
604
- tableName = tableName.substring(0, tableName.length() - 1);
605
- //if (!connection.tableExists(tableName)) {
606
- // TODO this doesn't help. Rather than truncating more characters,
607
- // here needs to replace characters with random characters. But
608
- // to make the result deterministic. So, an idea is replacing
609
- // the last character to the first (second, third, ... for each loop)
610
- // of md5(original table name).
611
- //}
612
- }
613
-
614
- }
615
- return tableName + "_" + uniqueSuffix;
616
- }
617
-
618
- private static JdbcSchema applyColumnOptionsToNewTableSchema(JdbcSchema schema, final Map<String, JdbcColumnOption> columnOptions)
619
- {
620
- return new JdbcSchema(Lists.transform(schema.getColumns(), new Function<JdbcColumn, JdbcColumn>() {
621
- public JdbcColumn apply(JdbcColumn c)
622
- {
623
- JdbcColumnOption option = columnOptionOf(columnOptions, c.getName());
624
- if (option.getType().isPresent()) {
625
- return JdbcColumn.newTypeDeclaredColumn(
626
- c.getName(), Types.OTHER, // sqlType, isNotNull, and isUniqueKey are ignored
627
- option.getType().get(), false, false);
628
- }
629
- return c;
630
- }
631
- }));
632
- }
633
-
634
- protected static List<ColumnSetter> newColumnSetters(ColumnSetterFactory factory,
635
- JdbcSchema targetTableSchema, Schema inputValueSchema,
636
- Map<String, JdbcColumnOption> columnOptions)
637
- {
638
- ImmutableList.Builder<ColumnSetter> builder = ImmutableList.builder();
639
- for (int schemaColumnIndex = 0; schemaColumnIndex < targetTableSchema.getCount(); schemaColumnIndex++) {
640
- JdbcColumn targetColumn = targetTableSchema.getColumn(schemaColumnIndex);
641
- Column inputColumn = inputValueSchema.getColumn(schemaColumnIndex);
642
- if (targetColumn.isSkipColumn()) {
643
- builder.add(factory.newSkipColumnSetter());
644
- } else {
645
- JdbcColumnOption option = columnOptionOf(columnOptions, inputColumn.getName());
646
- builder.add(factory.newColumnSetter(targetColumn, option));
647
- }
648
- }
649
- return builder.build();
650
- }
651
-
652
- private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, String columnName)
653
- {
654
- return Optional.fromNullable(columnOptions.get(columnName)).or(
655
- // default column option
656
- new Supplier<JdbcColumnOption>()
657
- {
658
- public JdbcColumnOption get()
659
- {
660
- return Exec.newConfigSource().loadConfig(JdbcColumnOption.class);
661
- }
662
- });
663
- }
664
-
665
- private boolean checkTableNameLength(String tableName, Charset tableNameCharset,
666
- int suffixLength, int maxLength, LengthSemantics lengthSemantics)
667
- {
668
- return lengthSemantics.countLength(tableNameCharset, tableName) + suffixLength <= maxLength;
669
- }
670
-
671
- protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
672
- throws SQLException
673
- {
674
- JdbcSchema schema = filterSkipColumns(task.getTargetTableSchema());
675
-
676
- switch (task.getMode()) {
677
- case INSERT_DIRECT:
678
- case MERGE_DIRECT:
679
- // already done
680
- if (task.getAfterLoad().isPresent()) {
681
- con.executeSql(task.getAfterLoad().get());
682
- }
683
- break;
684
-
685
- case INSERT:
686
- // aggregate insert into target
687
- if (task.getNewTableSchema().isPresent()) {
688
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
689
- }
690
- con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), false, task.getAfterLoad());
691
- break;
692
-
693
- case TRUNCATE_INSERT:
694
- // truncate & aggregate insert into target
695
- if (task.getNewTableSchema().isPresent()) {
696
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
697
- }
698
- con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), true, task.getAfterLoad());
699
- break;
700
-
701
- case MERGE:
702
- // aggregate merge into target
703
- if (task.getNewTableSchema().isPresent()) {
704
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
705
- }
706
- con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(),
707
- new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()), task.getAfterLoad());
708
- break;
709
-
710
- case REPLACE:
711
- // swap table
712
- con.replaceTable(task.getIntermediateTables().get().get(0), schema, task.getActualTable(), task.getAfterLoad());
713
- break;
714
- }
715
- }
716
-
717
- protected void doCleanup(JdbcOutputConnection con, PluginTask task, int taskCount,
718
- List<TaskReport> successTaskReports)
719
- throws SQLException
720
- {
721
- if (task.getIntermediateTables().isPresent()) {
722
- for (String intermTable : task.getIntermediateTables().get()) {
723
- con.dropTableIfExists(intermTable);
724
- }
725
- }
726
- }
727
-
728
- protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
729
- {
730
- final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
731
- for (Column c : schema.getColumns()) {
732
- final String columnName = c.getName();
733
- c.visit(new ColumnVisitor() {
734
- public void booleanColumn(Column column)
735
- {
736
- columns.add(JdbcColumn.newGenericTypeColumn(
737
- columnName, Types.BOOLEAN, "BOOLEAN",
738
- 1, 0, false, false));
739
- }
740
-
741
- public void longColumn(Column column)
742
- {
743
- columns.add(JdbcColumn.newGenericTypeColumn(
744
- columnName, Types.BIGINT, "BIGINT",
745
- 22, 0, false, false));
746
- }
747
-
748
- public void doubleColumn(Column column)
749
- {
750
- columns.add(JdbcColumn.newGenericTypeColumn(
751
- columnName, Types.FLOAT, "DOUBLE PRECISION",
752
- 24, 0, false, false));
753
- }
754
-
755
- // Teradata's specification: A LOB can not be the only column in a derived table
756
- // TODO: I want to get column type from system table
757
- public void stringColumn(Column column)
758
- {
759
- columns.add(JdbcColumn.newGenericTypeColumn(
760
- columnName, Types.CLOB, "CLOB",
761
- 4000, 0, false, false)); // TODO size type param
762
- }
763
-
764
- public void jsonColumn(Column column)
765
- {
766
- columns.add(JdbcColumn.newGenericTypeColumn(
767
- columnName, Types.CLOB, "CLOB",
768
- 4000, 0, false, false)); // TODO size type param
769
- }
770
-
771
- public void timestampColumn(Column column)
772
- {
773
- columns.add(JdbcColumn.newGenericTypeColumn(
774
- columnName, Types.TIMESTAMP, "TIMESTAMP",
775
- 26, 0, false, false)); // size type param is from postgresql
776
- }
777
- });
778
- }
779
- return new JdbcSchema(columns.build());
780
- }
781
-
782
- public Optional<JdbcSchema> newJdbcSchemaFromTableIfExists(JdbcOutputConnection connection,
783
- String tableName) throws SQLException
784
- {
785
- if (!connection.tableExists(tableName)) {
786
- // DatabaseMetaData.getPrimaryKeys fails if table does not exist
787
- return Optional.absent();
788
- }
789
-
790
- DatabaseMetaData dbm = connection.getMetaData();
791
- String escape = dbm.getSearchStringEscape();
792
-
793
- ResultSet rs = dbm.getPrimaryKeys(null, connection.getSchemaName(), tableName);
794
- ImmutableSet.Builder<String> primaryKeysBuilder = ImmutableSet.builder();
795
- try {
796
- while(rs.next()) {
797
- primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
798
- }
799
- } finally {
800
- rs.close();
801
- }
802
- ImmutableSet<String> primaryKeys = primaryKeysBuilder.build();
803
-
804
- ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
805
- rs = dbm.getColumns(null,
806
- JdbcUtils.escapeSearchString(connection.getSchemaName(), escape),
807
- JdbcUtils.escapeSearchString(tableName, escape),
808
- null);
809
- try {
810
- while (rs.next()) {
811
- String columnName = rs.getString("COLUMN_NAME");
812
- String simpleTypeName = rs.getString("TYPE_NAME").toUpperCase(Locale.ENGLISH);
813
- boolean isUniqueKey = primaryKeys.contains(columnName);
814
- int sqlType = rs.getInt("DATA_TYPE");
815
- int colSize = rs.getInt("COLUMN_SIZE");
816
- int decDigit = rs.getInt("DECIMAL_DIGITS");
817
- if (rs.wasNull()) {
818
- decDigit = -1;
819
- }
820
- int charOctetLength = rs.getInt("CHAR_OCTET_LENGTH");
821
- boolean isNotNull = "NO".equals(rs.getString("IS_NULLABLE"));
822
- //rs.getString("COLUMN_DEF") // or null // TODO
823
- builder.add(JdbcColumn.newGenericTypeColumn(
824
- columnName, sqlType, simpleTypeName, colSize, decDigit, charOctetLength, isNotNull, isUniqueKey));
825
- // We can't get declared column name using JDBC API.
826
- // Subclasses need to overwrite it.
827
- }
828
- } finally {
829
- rs.close();
830
- }
831
- List<JdbcColumn> columns = builder.build();
832
- if (columns.isEmpty()) {
833
- return Optional.absent();
834
- } else {
835
- return Optional.of(new JdbcSchema(columns));
836
- }
837
- }
838
-
839
- private JdbcSchema matchSchemaByColumnNames(Schema inputSchema, JdbcSchema targetTableSchema)
840
- {
841
- ImmutableList.Builder<JdbcColumn> jdbcColumns = ImmutableList.builder();
842
-
843
- for (Column column : inputSchema.getColumns()) {
844
- Optional<JdbcColumn> c = targetTableSchema.findColumn(column.getName());
845
- jdbcColumns.add(c.or(JdbcColumn.skipColumn()));
846
- }
847
-
848
- return new JdbcSchema(jdbcColumns.build());
849
- }
850
-
851
- public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final int taskIndex)
852
- {
853
- final PluginTask task = taskSource.loadTask(getTaskClass());
854
- final Mode mode = task.getMode();
855
-
856
- // instantiate BatchInsert without table name
857
- BatchInsert batch = null;
858
- try {
859
- Optional<MergeConfig> config = Optional.absent();
860
- if (task.getMode() == Mode.MERGE_DIRECT) {
861
- config = Optional.of(new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()));
862
- }
863
- batch = newBatchInsert(task, config);
864
- } catch (IOException | SQLException ex) {
865
- throw new RuntimeException(ex);
866
- }
867
-
868
- try {
869
- // configure PageReader -> BatchInsert
870
- PageReader reader = new PageReader(schema);
871
-
872
- List<ColumnSetter> columnSetters = newColumnSetters(
873
- newColumnSetterFactory(batch, task.getDefaultTimeZone()),
874
- task.getTargetTableSchema(), schema,
875
- task.getColumnOptions());
876
- JdbcSchema insertIntoSchema = filterSkipColumns(task.getTargetTableSchema());
877
- if (insertIntoSchema.getCount() == 0) {
878
- throw new SQLException("No column to insert.");
879
- }
880
-
881
- // configure BatchInsert -> an intermediate table (!isDirectModify) or the target table (isDirectModify)
882
- String destTable;
883
- if (mode.tempTablePerTask()) {
884
- destTable = task.getIntermediateTables().get().get(taskIndex);
885
- } else if (mode.isDirectModify()) {
886
- destTable = task.getActualTable();
887
- } else {
888
- destTable = task.getIntermediateTables().get().get(0);
889
- }
890
- batch.prepare(destTable, insertIntoSchema);
891
-
892
- PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize(), task);
893
- batch = null;
894
- return output;
895
-
896
- } catch (SQLException ex) {
897
- throw new RuntimeException(ex);
898
-
899
- } finally {
900
- if (batch != null) {
901
- try {
902
- batch.close();
903
- } catch (IOException | SQLException ex) {
904
- throw new RuntimeException(ex);
905
- }
906
- }
907
- }
908
- }
909
-
910
- public class PluginPageOutput
911
- implements TransactionalPageOutput
912
- {
913
- protected final List<Column> columns;
914
- protected final List<ColumnSetterVisitor> columnVisitors;
915
- private final PageReader pageReader;
916
- private final BatchInsert batch;
917
- private final int batchSize;
918
- private final int forceBatchFlushSize;
919
- private final PluginTask task;
920
-
921
- public PluginPageOutput(final PageReader pageReader,
922
- BatchInsert batch, List<ColumnSetter> columnSetters,
923
- int batchSize, PluginTask task)
924
- {
925
- this.pageReader = pageReader;
926
- this.batch = batch;
927
- this.columns = pageReader.getSchema().getColumns();
928
- this.columnVisitors = ImmutableList.copyOf(Lists.transform(
929
- columnSetters, new Function<ColumnSetter, ColumnSetterVisitor>() {
930
- public ColumnSetterVisitor apply(ColumnSetter setter)
931
- {
932
- return new ColumnSetterVisitor(pageReader, setter);
933
- }
934
- }));
935
- this.batchSize = batchSize;
936
- this.task = task;
937
- this.forceBatchFlushSize = batchSize * 2;
938
- }
939
-
940
- @Override
941
- public void add(Page page)
942
- {
943
- try {
944
- pageReader.setPage(page);
945
- while (pageReader.nextRecord()) {
946
- if (batch.getBatchWeight() > forceBatchFlushSize) {
947
- withRetry(task, new IdempotentSqlRunnable() {
948
- @Override
949
- public void run() throws SQLException {
950
- try {
951
- batch.flush();
952
- } catch (IOException ex) {
953
- throw new RuntimeException(ex);
954
- }
955
- }
956
- });
957
- }
958
- handleColumnsSetters();
959
- batch.add();
960
- }
961
- if (batch.getBatchWeight() > batchSize) {
962
- withRetry(task, new IdempotentSqlRunnable() {
963
- @Override
964
- public void run() throws SQLException {
965
- try {
966
- batch.flush();
967
- } catch (IOException ex) {
968
- throw new RuntimeException(ex);
969
- }
970
- }
971
- });
972
- }
973
- } catch (IOException | SQLException | InterruptedException ex) {
974
- throw new RuntimeException(ex);
975
- }
976
- }
977
-
978
- @Override
979
- public void finish()
980
- {
981
- try {
982
- withRetry(task, new IdempotentSqlRunnable() {
983
- @Override
984
- public void run() throws SQLException {
985
- try {
986
- batch.finish();
987
- } catch (IOException ex) {
988
- throw new RuntimeException(ex);
989
- }
990
- }
991
- });
992
- } catch (InterruptedException | SQLException ex) {
993
- throw new RuntimeException(ex);
994
- }
995
- }
996
-
997
- @Override
998
- public void close()
999
- {
1000
- try {
1001
- batch.close();
1002
- } catch (IOException | SQLException ex) {
1003
- throw new RuntimeException(ex);
1004
- }
1005
- }
1006
-
1007
- @Override
1008
- public void abort()
1009
- {
1010
- }
1011
-
1012
- @Override
1013
- public TaskReport commit()
1014
- {
1015
- return Exec.newTaskReport();
1016
- }
1017
-
1018
- protected void handleColumnsSetters()
1019
- {
1020
- int size = columnVisitors.size();
1021
- for (int i=0; i < size; i++) {
1022
- columns.get(i).visit(columnVisitors.get(i));
1023
- }
1024
- }
1025
- }
1026
-
1027
- protected boolean isRetryableException(SQLException exception)
1028
- {
1029
- return isRetryableException(exception.getSQLState(), exception.getErrorCode());
1030
- }
1031
- protected boolean isRetryableException(String sqlState, int errorCode)
1032
- {
1033
- return false;
1034
- }
1035
-
1036
-
1037
- public static interface IdempotentSqlRunnable
1038
- {
1039
- public void run() throws SQLException;
1040
- }
1041
-
1042
- protected void withRetry(PluginTask task, IdempotentSqlRunnable op)
1043
- throws SQLException, InterruptedException
1044
- {
1045
- withRetry(task, op, "Operation failed");
1046
- }
1047
-
1048
- protected void withRetry(PluginTask task, final IdempotentSqlRunnable op, final String errorMessage)
1049
- throws SQLException, InterruptedException
1050
- {
1051
- try {
1052
- buildRetryExecutor(task)
1053
- .runInterruptible(new RetryableSQLExecution(op, errorMessage));
1054
- } catch (ExecutionException ex) {
1055
- Throwable cause = ex.getCause();
1056
- Throwables.propagateIfInstanceOf(cause, SQLException.class);
1057
- throw Throwables.propagate(cause);
1058
- }
1059
- }
1060
-
1061
- private static RetryExecutor buildRetryExecutor(PluginTask task) {
1062
- return retryExecutor()
1063
- .withRetryLimit(task.getRetryLimit())
1064
- .withInitialRetryWait(task.getRetryWait())
1065
- .withMaxRetryWait(task.getMaxRetryWait());
1066
- }
1067
-
1068
- class RetryableSQLExecution implements Retryable<Void> {
1069
- private final String errorMessage;
1070
- private final IdempotentSqlRunnable op;
1071
-
1072
- private final Logger logger = Exec.getLogger(this.getClass());
1073
-
1074
- public RetryableSQLExecution(IdempotentSqlRunnable op, String errorMessage) {
1075
- this.errorMessage = errorMessage;
1076
- this.op = op;
1077
- }
1078
-
1079
- public Void call() throws Exception {
1080
- op.run();
1081
- return null;
1082
- }
1083
-
1084
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
1085
- {
1086
- if (exception instanceof SQLException) {
1087
- SQLException ex = (SQLException) exception;
1088
- String sqlState = ex.getSQLState();
1089
- int errorCode = ex.getErrorCode();
1090
- logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
1091
- errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
1092
- buildExceptionMessage(exception));
1093
- } else {
1094
- logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
1095
- errorMessage, retryCount, retryLimit, retryWait/1000,
1096
- buildExceptionMessage(exception));
1097
- }
1098
- if (retryCount % 3 == 0) {
1099
- logger.info("Error details:", exception);
1100
- }
1101
- }
1102
-
1103
- public void onGiveup(Exception firstException, Exception lastException)
1104
- {
1105
- if (firstException instanceof SQLException) {
1106
- SQLException ex = (SQLException) firstException;
1107
- String sqlState = ex.getSQLState();
1108
- int errorCode = ex.getErrorCode();
1109
- logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
1110
- }
1111
- }
1112
-
1113
- public boolean isRetryableException(Exception exception) {
1114
- if (exception instanceof SQLException) {
1115
- SQLException ex = (SQLException)exception;
1116
-
1117
- return AbstractJdbcOutputPlugin.this.isRetryableException(ex);
1118
- } else {
1119
- return false;
1120
- }
1121
- }
1122
-
1123
- private String buildExceptionMessage(Throwable ex) {
1124
- StringBuilder sb = new StringBuilder();
1125
- sb.append(ex.getMessage());
1126
- if (ex.getCause() != null) {
1127
- buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
1128
- }
1129
- return sb.toString();
1130
- }
1131
-
1132
- private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
1133
- if (!lastMessage.equals(ex.getMessage())) {
1134
- // suppress same messages
1135
- sb.append(" < ");
1136
- sb.append(ex.getMessage());
1137
- }
1138
- if (ex.getCause() == null) {
1139
- return;
1140
- }
1141
- buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
1142
- }
1143
- }
1144
- }