embulk-output-jdbc 0.2.3 → 0.2.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/build.gradle +2 -2
  3. data/classpath/{embulk-output-jdbc-0.2.3.jar → embulk-output-jdbc-0.2.4.jar} +0 -0
  4. data/lib/embulk/output/jdbc.rb +3 -3
  5. data/src/main/java/org/embulk/output/JdbcOutputPlugin.java +120 -120
  6. data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +755 -756
  7. data/src/main/java/org/embulk/output/jdbc/BatchInsert.java +54 -54
  8. data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnector.java +8 -8
  9. data/src/main/java/org/embulk/output/jdbc/JdbcSchema.java +37 -37
  10. data/src/main/java/org/embulk/output/jdbc/JdbcUtils.java +155 -155
  11. data/src/main/java/org/embulk/output/jdbc/RetryExecutor.java +105 -105
  12. data/src/main/java/org/embulk/output/jdbc/setter/BooleanColumnSetter.java +52 -52
  13. data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetterFactory.java +137 -137
  14. data/src/main/java/org/embulk/output/jdbc/setter/DoubleColumnSetter.java +51 -51
  15. data/src/main/java/org/embulk/output/jdbc/setter/LongColumnSetter.java +62 -62
  16. data/src/main/java/org/embulk/output/jdbc/setter/NullColumnSetter.java +43 -43
  17. data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java +38 -38
  18. data/src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java +48 -48
  19. data/src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java +48 -48
  20. data/src/test/java/org/embulk/output/TestJdbcOutputPlugin.java +5 -5
  21. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ed0f3d732ac1fa2ee1d660480be8587ac137d3f0
4
- data.tar.gz: c534795f00578dc8a30e580f9b34a8252e5cc5d4
3
+ metadata.gz: a3248904e8b007fc46013b4697d939d7dd3f55d8
4
+ data.tar.gz: f3eacb5f104d9ea4f474e14a7c0ea724c9831341
5
5
  SHA512:
6
- metadata.gz: 57157af5fd5be3ceeab92d8dbf36c9f031e2a1ed5bb8f2ae78bf0d91629efdcaf0523de04bbeed799633f40596c4159d8ef589c267b0693d6acd591973b6ee1c
7
- data.tar.gz: cbfa1e7646587f442a233f946bde40fe3525414713727333bbe98720408d71067b7e2e60e88358d9d86ac367ed58d1744f4f876e2e7357f5ae3f1009ac5c3786
6
+ metadata.gz: 5d8f93e2f5f014df77b5675ce58209e1c17a43b02e17a4dbe5fc57348461a90659a5aacd8c61f8f68bba94c08cab0da2facd6989c57b90e8357c810d2c3f84be
7
+ data.tar.gz: 6afcba3953c431e6c0fc4e5733a91c4dff9b4c04af6d5eb04869aeb7b8c2d432f2df8ddc6a96ae725bfeae18cb62f413be206f9dcad2246391de45cbfbe87c5b
data/build.gradle CHANGED
@@ -1,2 +1,2 @@
1
- dependencies {
2
- }
1
+ dependencies {
2
+ }
data/lib/embulk/output/jdbc.rb CHANGED
@@ -1,3 +1,3 @@
1
- Embulk::JavaPlugin.register_output(
2
- :jdbc, "org.embulk.output.JdbcOutputPlugin",
3
- File.expand_path('../../../../classpath', __FILE__))
1
+ Embulk::JavaPlugin.register_output(
2
+ :jdbc, "org.embulk.output.JdbcOutputPlugin",
3
+ File.expand_path('../../../../classpath', __FILE__))
data/src/main/java/org/embulk/output/JdbcOutputPlugin.java CHANGED
@@ -1,120 +1,120 @@
1
- package org.embulk.output;
2
-
3
- import java.util.Properties;
4
- import java.sql.Driver;
5
- import java.io.IOException;
6
- import java.sql.Connection;
7
- import java.sql.SQLException;
8
- import com.google.common.base.Optional;
9
- import com.google.common.base.Throwables;
10
- import org.embulk.config.Config;
11
- import org.embulk.config.ConfigDefault;
12
- import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
13
- import org.embulk.output.jdbc.BatchInsert;
14
- import org.embulk.output.jdbc.StandardBatchInsert;
15
- import org.embulk.output.jdbc.JdbcOutputConnector;
16
- import org.embulk.output.jdbc.JdbcOutputConnection;
17
-
18
- public class JdbcOutputPlugin
19
- extends AbstractJdbcOutputPlugin
20
- {
21
- public interface GenericPluginTask extends PluginTask
22
- {
23
- @Config("driver_path")
24
- @ConfigDefault("null")
25
- public Optional<String> getDriverPath();
26
-
27
- @Config("driver_class")
28
- public String getDriverClass();
29
-
30
- @Config("url")
31
- public String getUrl();
32
-
33
- @Config("user")
34
- @ConfigDefault("null")
35
- public Optional<String> getUser();
36
-
37
- @Config("password")
38
- @ConfigDefault("null")
39
- public Optional<String> getPassword();
40
-
41
- @Config("schema")
42
- @ConfigDefault("null")
43
- public Optional<String> getSchema();
44
- }
45
-
46
- @Override
47
- protected Class<? extends PluginTask> getTaskClass()
48
- {
49
- return GenericPluginTask.class;
50
- }
51
-
52
- @Override
53
- protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
54
- {
55
- GenericPluginTask t = (GenericPluginTask) task;
56
-
57
- if (t.getDriverPath().isPresent()) {
58
- loadDriverJar(t.getDriverPath().get());
59
- }
60
-
61
- Properties props = new Properties();
62
- if (t.getUser().isPresent()) {
63
- props.setProperty("user", t.getUser().get());
64
- }
65
- if (t.getPassword().isPresent()) {
66
- props.setProperty("password", t.getPassword().get());
67
- }
68
-
69
- props.putAll(t.getOptions());
70
-
71
- return new GenericOutputConnector(t.getUrl(), props, t.getDriverClass(),
72
- t.getSchema().orNull());
73
- }
74
-
75
- private static class GenericOutputConnector
76
- implements JdbcOutputConnector
77
- {
78
- private final Driver driver;
79
- private final String url;
80
- private final Properties properties;
81
- private final String schemaName;
82
-
83
- public GenericOutputConnector(String url, Properties properties, String driverClass,
84
- String schemaName)
85
- {
86
- try {
87
- // TODO check Class.forName(driverClass) is a Driver before newInstance
88
- // for security
89
- this.driver = (Driver) Class.forName(driverClass).newInstance();
90
- } catch (Exception ex) {
91
- throw Throwables.propagate(ex);
92
- }
93
- this.url = url;
94
- this.properties = properties;
95
- this.schemaName = schemaName;
96
- }
97
-
98
- @Override
99
- public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
100
- {
101
- Connection c = driver.connect(url, properties);
102
- try {
103
- c.setAutoCommit(autoCommit);
104
- JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
105
- c = null;
106
- return con;
107
- } finally {
108
- if (c != null) {
109
- c.close();
110
- }
111
- }
112
- }
113
- }
114
-
115
- @Override
116
- protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
117
- {
118
- return new StandardBatchInsert(getConnector(task, true));
119
- }
120
- }
1
+ package org.embulk.output;
2
+
3
+ import java.util.Properties;
4
+ import java.sql.Driver;
5
+ import java.io.IOException;
6
+ import java.sql.Connection;
7
+ import java.sql.SQLException;
8
+ import com.google.common.base.Optional;
9
+ import com.google.common.base.Throwables;
10
+ import org.embulk.config.Config;
11
+ import org.embulk.config.ConfigDefault;
12
+ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
13
+ import org.embulk.output.jdbc.BatchInsert;
14
+ import org.embulk.output.jdbc.StandardBatchInsert;
15
+ import org.embulk.output.jdbc.JdbcOutputConnector;
16
+ import org.embulk.output.jdbc.JdbcOutputConnection;
17
+
18
+ public class JdbcOutputPlugin
19
+ extends AbstractJdbcOutputPlugin
20
+ {
21
+ public interface GenericPluginTask extends PluginTask
22
+ {
23
+ @Config("driver_path")
24
+ @ConfigDefault("null")
25
+ public Optional<String> getDriverPath();
26
+
27
+ @Config("driver_class")
28
+ public String getDriverClass();
29
+
30
+ @Config("url")
31
+ public String getUrl();
32
+
33
+ @Config("user")
34
+ @ConfigDefault("null")
35
+ public Optional<String> getUser();
36
+
37
+ @Config("password")
38
+ @ConfigDefault("null")
39
+ public Optional<String> getPassword();
40
+
41
+ @Config("schema")
42
+ @ConfigDefault("null")
43
+ public Optional<String> getSchema();
44
+ }
45
+
46
+ @Override
47
+ protected Class<? extends PluginTask> getTaskClass()
48
+ {
49
+ return GenericPluginTask.class;
50
+ }
51
+
52
+ @Override
53
+ protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
54
+ {
55
+ GenericPluginTask t = (GenericPluginTask) task;
56
+
57
+ if (t.getDriverPath().isPresent()) {
58
+ loadDriverJar(t.getDriverPath().get());
59
+ }
60
+
61
+ Properties props = new Properties();
62
+ if (t.getUser().isPresent()) {
63
+ props.setProperty("user", t.getUser().get());
64
+ }
65
+ if (t.getPassword().isPresent()) {
66
+ props.setProperty("password", t.getPassword().get());
67
+ }
68
+
69
+ props.putAll(t.getOptions());
70
+
71
+ return new GenericOutputConnector(t.getUrl(), props, t.getDriverClass(),
72
+ t.getSchema().orNull());
73
+ }
74
+
75
+ private static class GenericOutputConnector
76
+ implements JdbcOutputConnector
77
+ {
78
+ private final Driver driver;
79
+ private final String url;
80
+ private final Properties properties;
81
+ private final String schemaName;
82
+
83
+ public GenericOutputConnector(String url, Properties properties, String driverClass,
84
+ String schemaName)
85
+ {
86
+ try {
87
+ // TODO check Class.forName(driverClass) is a Driver before newInstance
88
+ // for security
89
+ this.driver = (Driver) Class.forName(driverClass).newInstance();
90
+ } catch (Exception ex) {
91
+ throw Throwables.propagate(ex);
92
+ }
93
+ this.url = url;
94
+ this.properties = properties;
95
+ this.schemaName = schemaName;
96
+ }
97
+
98
+ @Override
99
+ public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
100
+ {
101
+ Connection c = driver.connect(url, properties);
102
+ try {
103
+ c.setAutoCommit(autoCommit);
104
+ JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
105
+ c = null;
106
+ return con;
107
+ } finally {
108
+ if (c != null) {
109
+ c.close();
110
+ }
111
+ }
112
+ }
113
+ }
114
+
115
+ @Override
116
+ protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
117
+ {
118
+ return new StandardBatchInsert(getConnector(task, true));
119
+ }
120
+ }
data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java CHANGED
@@ -1,756 +1,755 @@
1
- package org.embulk.output.jdbc;
2
-
3
- import java.util.HashSet;
4
- import java.util.List;
5
- import java.util.Locale;
6
- import java.util.Properties;
7
- import java.util.Set;
8
- import java.util.concurrent.ExecutionException;
9
- import java.io.IOException;
10
- import java.nio.file.Paths;
11
- import java.sql.Types;
12
- import java.sql.ResultSet;
13
- import java.sql.DatabaseMetaData;
14
- import java.sql.SQLException;
15
-
16
- import org.slf4j.Logger;
17
-
18
- import com.google.common.base.Optional;
19
- import com.google.common.base.Throwables;
20
- import com.google.common.collect.ImmutableList;
21
-
22
- import org.embulk.config.CommitReport;
23
- import org.embulk.config.Config;
24
- import org.embulk.config.ConfigDefault;
25
- import org.embulk.config.ConfigDiff;
26
- import org.embulk.config.ConfigException;
27
- import org.embulk.config.ConfigSource;
28
- import org.embulk.config.Task;
29
- import org.embulk.config.TaskSource;
30
- import org.embulk.spi.Exec;
31
- import org.embulk.spi.Column;
32
- import org.embulk.spi.ColumnVisitor;
33
- import org.embulk.spi.OutputPlugin;
34
- import org.embulk.spi.PluginClassLoader;
35
- import org.embulk.spi.Schema;
36
- import org.embulk.spi.TransactionalPageOutput;
37
- import org.embulk.spi.Page;
38
- import org.embulk.spi.PageReader;
39
- import org.embulk.spi.time.Timestamp;
40
- import org.embulk.spi.time.TimestampFormatter;
41
- import org.embulk.output.jdbc.setter.ColumnSetter;
42
- import org.embulk.output.jdbc.setter.ColumnSetterFactory;
43
- import org.embulk.output.jdbc.RetryExecutor.IdempotentOperation;
44
-
45
- import static org.embulk.output.jdbc.RetryExecutor.retryExecutor;
46
-
47
- public abstract class AbstractJdbcOutputPlugin
48
- implements OutputPlugin
49
- {
50
- private final static Set<String> loadedJarGlobs = new HashSet<String>();
51
-
52
- private final Logger logger = Exec.getLogger(getClass());
53
-
54
- public interface PluginTask
55
- extends Task
56
- {
57
- @Config("options")
58
- @ConfigDefault("{}")
59
- public Properties getOptions();
60
-
61
- @Config("table")
62
- public String getTable();
63
-
64
- @Config("mode")
65
- public String getModeConfig();
66
-
67
- @Config("batch_size")
68
- @ConfigDefault("16777216")
69
- // TODO set minimum number
70
- public int getBatchSize();
71
-
72
- public void setMode(Mode mode);
73
- public Mode getMode();
74
-
75
- public JdbcSchema getLoadSchema();
76
- public void setLoadSchema(JdbcSchema schema);
77
-
78
- public Optional<String> getSwapTable();
79
- public void setSwapTable(Optional<String> name);
80
-
81
- public Optional<String> getMultipleLoadTablePrefix();
82
- public void setMultipleLoadTablePrefix(Optional<String> prefix);
83
- }
84
-
85
- protected void loadDriverJar(String glob)
86
- {
87
- synchronized (loadedJarGlobs) {
88
- if (!loadedJarGlobs.contains(glob)) {
89
- // TODO match glob
90
- PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
91
- loader.addPath(Paths.get(glob));
92
- loadedJarGlobs.add(glob);
93
- }
94
- }
95
- }
96
-
97
- // for subclasses to add @Config
98
- protected Class<? extends PluginTask> getTaskClass()
99
- {
100
- return PluginTask.class;
101
- }
102
-
103
- protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
104
-
105
- protected abstract BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException;
106
-
107
- protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
108
- boolean autoCommit) throws SQLException
109
- {
110
- return getConnector(task, retryableMetadataOperation).connect(autoCommit);
111
- }
112
-
113
- public enum Mode {
114
- INSERT,
115
- INSERT_DIRECT,
116
- TRUNCATE_INSERT,
117
- MERGE,
118
- REPLACE,
119
- REPLACE_INPLACE;
120
- //REPLACE_PARTITIONING, // MySQL: partitioning, PostgreSQL: inheritance
121
-
122
- public boolean isDirectWrite()
123
- {
124
- return this == INSERT_DIRECT;
125
- }
126
-
127
- public boolean isInplace()
128
- {
129
- return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
130
- }
131
-
132
- public boolean isMerge()
133
- {
134
- return this == MERGE;
135
- }
136
-
137
- public boolean usesMultipleLoadTables()
138
- {
139
- return !isInplace();
140
- }
141
-
142
- public boolean createAndSwapTable()
143
- {
144
- return this == REPLACE_INPLACE || this == REPLACE;
145
- }
146
- }
147
-
148
- public ConfigDiff transaction(ConfigSource config,
149
- Schema schema, int taskCount,
150
- OutputPlugin.Control control)
151
- {
152
- PluginTask task = config.loadConfig(getTaskClass());
153
-
154
- // TODO this is a temporary code. behavior will change in a future release.
155
- switch(task.getModeConfig()) {
156
- case "insert":
157
- task.setMode(Mode.INSERT_DIRECT);
158
- break;
159
- case "replace":
160
- task.setMode(Mode.REPLACE_INPLACE);
161
- break;
162
- case "merge":
163
- task.setMode(Mode.MERGE);
164
- break;
165
- default:
166
- throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
167
- }
168
-
169
- //switch(task.getModeConfig()) {
170
- ////case "insert":
171
- //// task.setMode(Mode.INSERT);
172
- //// break;
173
- //case "insert_direct":
174
- // task.setMode(Mode.INSERT_DIRECT);
175
- // break;
176
- ////case "truncate_insert": // TODO
177
- //// task.setMode(Mode.TRUNCATE_INSERT);
178
- //// break;
179
- ////case "merge": // TODO
180
- //// task.setMode(Mode.MERGE);
181
- //// break;
182
- ////case "replace":
183
- //// task.setMode(Mode.REPLACE);
184
- //// break;
185
- //case "replace_inplace":
186
- // task.setMode(Mode.REPLACE_INPLACE);
187
- // break;
188
- //default:
189
- // new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert_direct, replace_inplace", task.getModeConfig()));
190
- //}
191
-
192
- task = begin(task, schema, taskCount);
193
- control.run(task.dump());
194
- return commit(task, schema, taskCount);
195
- }
196
-
197
- public ConfigDiff resume(TaskSource taskSource,
198
- Schema schema, int taskCount,
199
- OutputPlugin.Control control)
200
- {
201
- PluginTask task = taskSource.loadTask(getTaskClass());
202
-
203
- if (task.getMode().isInplace()) {
204
- throw new UnsupportedOperationException("inplace mode is not resumable. You need to delete partially-loaded records from the database and restart the entire transaction.");
205
- }
206
-
207
- task = begin(task, schema, taskCount);
208
- control.run(task.dump());
209
- return commit(task, schema, taskCount);
210
- }
211
-
212
- protected String getTransactionUniqueName()
213
- {
214
- // TODO use uuid?
215
- Timestamp t = Exec.session().getTransactionTime();
216
- return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
217
- }
218
-
219
- private PluginTask begin(final PluginTask task,
220
- final Schema schema, int taskCount)
221
- {
222
- try {
223
- withRetry(new IdempotentSqlRunnable() { // no intermediate data if isDirectWrite == true
224
- public void run() throws SQLException
225
- {
226
- JdbcOutputConnection con = newConnection(task, true, false);
227
- try {
228
- doBegin(con, task, schema);
229
- } finally {
230
- con.close();
231
- }
232
- }
233
- });
234
- } catch (SQLException | InterruptedException ex) {
235
- throw new RuntimeException(ex);
236
- }
237
- return task;
238
- }
239
-
240
- private ConfigDiff commit(final PluginTask task,
241
- Schema schema, final int taskCount)
242
- {
243
- if (!task.getMode().isDirectWrite()) { // no intermediate data if isDirectWrite == true
244
- try {
245
- withRetry(new IdempotentSqlRunnable() {
246
- public void run() throws SQLException
247
- {
248
- JdbcOutputConnection con = newConnection(task, false, false);
249
- try {
250
- doCommit(con, task, taskCount);
251
- } finally {
252
- con.close();
253
- }
254
- }
255
- });
256
- } catch (SQLException | InterruptedException ex) {
257
- throw new RuntimeException(ex);
258
- }
259
- }
260
- return Exec.newConfigDiff();
261
- }
262
-
263
- public void cleanup(TaskSource taskSource,
264
- Schema schema, final int taskCount,
265
- final List<CommitReport> successCommitReports)
266
- {
267
- final PluginTask task = taskSource.loadTask(getTaskClass());
268
-
269
- if (!task.getMode().isDirectWrite()) { // no intermediate data if isDirectWrite == true
270
- try {
271
- withRetry(new IdempotentSqlRunnable() {
272
- public void run() throws SQLException
273
- {
274
- JdbcOutputConnection con = newConnection(task, true, true);
275
- try {
276
- doCleanup(con, task, taskCount, successCommitReports);
277
- } finally {
278
- con.close();
279
- }
280
- }
281
- });
282
- } catch (SQLException | InterruptedException ex) {
283
- throw new RuntimeException(ex);
284
- }
285
- }
286
- }
287
-
288
- protected void doBegin(JdbcOutputConnection con,
289
- PluginTask task, Schema schema) throws SQLException
290
- {
291
- Mode mode = task.getMode();
292
-
293
- JdbcSchema targetTableSchema;
294
- if (mode.createAndSwapTable()) {
295
- // DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
296
- // CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
297
- // swapTableName = "xyz__0000000054d92dee1e452158_bulk_load_temp"
298
- String swapTableName = generateSwapTableName(task);
299
- con.dropTableIfExists(swapTableName);
300
- con.createTableIfNotExists(swapTableName, newJdbcSchemaForNewTable(schema));
301
- targetTableSchema = newJdbcSchemaFromExistentTable(con, swapTableName);
302
- task.setSwapTable(Optional.of(swapTableName));
303
- } else {
304
- // CREATE TABLE IF NOT EXISTS xyz
305
- con.createTableIfNotExists(task.getTable(), newJdbcSchemaForNewTable(schema));
306
- targetTableSchema = newJdbcSchemaFromExistentTable(con, task.getTable());
307
- task.setSwapTable(Optional.<String>absent());
308
- }
309
-
310
- if (mode.usesMultipleLoadTables()) {
311
- // multipleLoadTablePrefix = "xyz__0000000054d92dee1e452158_"
312
- // workers run:
313
- // CREATE TABLE xyz__0000000054d92dee1e452158_%d
314
- String multipleLoadTablePrefix = task.getTable() + "_" + getTransactionUniqueName();
315
- task.setMultipleLoadTablePrefix(Optional.of(multipleLoadTablePrefix));
316
- } else {
317
- task.setMultipleLoadTablePrefix(Optional.<String>absent());
318
- }
319
-
320
- task.setLoadSchema(matchSchemaByColumnNames(schema, targetTableSchema));
321
- }
322
-
323
- protected String generateSwapTableName(PluginTask task) throws SQLException
324
- {
325
- return task.getTable() + "_" + getTransactionUniqueName() + "_bulk_load_temp";
326
- }
327
-
328
- protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
329
- throws SQLException
330
- {
331
- switch (task.getMode()) {
332
- case INSERT:
333
- // aggregate insert into target
334
- //con.gatherInsertTables();
335
- throw new UnsupportedOperationException("not implemented yet"); // TODO
336
- case INSERT_DIRECT:
337
- // already done
338
- break;
339
- case TRUNCATE_INSERT:
340
- // truncate & aggregate insert into target
341
- throw new UnsupportedOperationException("not implemented yet");
342
- //break;
343
- case MERGE:
344
- // aggregate merge into target
345
- throw new UnsupportedOperationException("not implemented yet");
346
- //break;
347
- case REPLACE:
348
- if (taskCount == 1) {
349
- // swap table
350
- con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
351
- } else {
352
- // aggregate insert into swap table & swap table
353
- throw new UnsupportedOperationException("not implemented yet");
354
- }
355
- break;
356
- case REPLACE_INPLACE:
357
- // swap table
358
- con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
359
- break;
360
- }
361
- }
362
-
363
- protected void doCleanup(JdbcOutputConnection con, PluginTask task, int taskCount,
364
- List<CommitReport> successCommitReports)
365
- throws SQLException
366
- {
367
- if (task.getSwapTable().isPresent()) {
368
- con.dropTableIfExists(task.getSwapTable().get());
369
- }
370
- if (task.getMultipleLoadTablePrefix().isPresent()) {
371
- for (int i=0; i < taskCount; i++) {
372
- con.dropTableIfExists(formatMultipleLoadTableName(task, i));
373
- }
374
- }
375
- }
376
-
377
- static String formatMultipleLoadTableName(PluginTask task, int taskIndex)
378
- {
379
- return task.getMultipleLoadTablePrefix().get() + String.format("%04x", taskIndex);
380
- }
381
-
382
- protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
383
- {
384
- final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
385
- for (Column c : schema.getColumns()) {
386
- final String columnName = c.getName();
387
- c.visit(new ColumnVisitor() {
388
- public void booleanColumn(Column column)
389
- {
390
- columns.add(new JdbcColumn(
391
- columnName, "BOOLEAN",
392
- Types.BOOLEAN, 1, 0, false));
393
- }
394
-
395
- public void longColumn(Column column)
396
- {
397
- columns.add(new JdbcColumn(
398
- columnName, "BIGINT",
399
- Types.BIGINT, 22, 0, false));
400
- }
401
-
402
- public void doubleColumn(Column column)
403
- {
404
- columns.add(new JdbcColumn(
405
- columnName, "DOUBLE PRECISION",
406
- Types.FLOAT, 24, 0, false));
407
- }
408
-
409
- public void stringColumn(Column column)
410
- {
411
- columns.add(new JdbcColumn(
412
- columnName, "CLOB",
413
- Types.CLOB, 4000, 0, false)); // TODO size type param
414
- }
415
-
416
- public void timestampColumn(Column column)
417
- {
418
- columns.add(new JdbcColumn(
419
- columnName, "TIMESTAMP",
420
- Types.TIMESTAMP, 26, 0, false)); // size type param is from postgresql.
421
- }
422
- });
423
- }
424
- return new JdbcSchema(columns.build());
425
- }
426
-
427
- public JdbcSchema newJdbcSchemaFromExistentTable(JdbcOutputConnection connection,
428
- String tableName) throws SQLException
429
- {
430
- DatabaseMetaData dbm = connection.getMetaData();
431
- String escape = dbm.getSearchStringEscape();
432
- String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
433
-
434
- ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
435
- ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
436
- try {
437
- while(rs.next()) {
438
- primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
439
- }
440
- } finally {
441
- rs.close();
442
- }
443
- ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
444
-
445
- String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
446
- ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
447
- rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
448
- try {
449
- while(rs.next()) {
450
- String columnName = rs.getString("COLUMN_NAME");
451
- String typeName = rs.getString("TYPE_NAME");
452
- boolean isPrimaryKey = primaryKeys.contains(columnName);
453
- typeName = typeName.toUpperCase(Locale.ENGLISH);
454
- int sqlType = rs.getInt("DATA_TYPE");
455
- int colSize = rs.getInt("COLUMN_SIZE");
456
- int decDigit = rs.getInt("DECIMAL_DIGITS");
457
- if (rs.wasNull()) {
458
- decDigit = -1;
459
- }
460
- //rs.getString("IS_NULLABLE").equals("NO") // "YES" or "" // TODO
461
- //rs.getString("COLUMN_DEF") // or null // TODO
462
- columns.add(new JdbcColumn(
463
- columnName, typeName,
464
- sqlType, colSize, decDigit, isPrimaryKey));
465
- }
466
- } finally {
467
- rs.close();
468
- }
469
- return new JdbcSchema(columns.build());
470
- }
471
-
472
- private JdbcSchema matchSchemaByColumnNames(Schema inputSchema, JdbcSchema targetTableSchema)
473
- {
474
- ImmutableList.Builder<JdbcColumn> jdbcColumns = ImmutableList.builder();
475
-
476
- outer : for (Column column : inputSchema.getColumns()) {
477
- for (JdbcColumn jdbcColumn : targetTableSchema.getColumns()) {
478
- if (jdbcColumn.getName().equals(column.getName())) {
479
- jdbcColumns.add(jdbcColumn);
480
- continue outer;
481
- }
482
- }
483
-
484
- jdbcColumns.add(JdbcColumn.skipColumn());
485
- }
486
-
487
- return new JdbcSchema(jdbcColumns.build());
488
- }
489
-
490
- public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final int taskIndex)
491
- {
492
- final PluginTask task = taskSource.loadTask(getTaskClass());
493
- final Mode mode = task.getMode();
494
-
495
- BatchInsert batch;
496
- try {
497
- batch = newBatchInsert(task);
498
- } catch (IOException | SQLException ex) {
499
- throw new RuntimeException(ex);
500
- }
501
- try {
502
- PageReader reader = new PageReader(schema);
503
- ColumnSetterFactory factory = newColumnSetterFactory(batch, reader, null); // TODO TimestampFormatter
504
-
505
- JdbcSchema loadSchema = task.getLoadSchema();
506
-
507
- ImmutableList.Builder<JdbcColumn> insertColumns = ImmutableList.builder();
508
- ImmutableList.Builder<ColumnSetter> columnSetters = ImmutableList.builder();
509
- for (JdbcColumn c : loadSchema.getColumns()) {
510
- if (c.isSkipColumn()) {
511
- columnSetters.add(factory.newSkipColumnSetter());
512
- } else {
513
- columnSetters.add(factory.newColumnSetter(c));
514
- insertColumns.add(c);
515
- }
516
- }
517
- final JdbcSchema insertSchema = new JdbcSchema(insertColumns.build());
518
-
519
- final BatchInsert b = batch;
520
- withRetry(new IdempotentSqlRunnable() {
521
- public void run() throws SQLException
522
- {
523
- String loadTable;
524
- boolean createTable;
525
- if (mode.usesMultipleLoadTables()) {
526
- // insert, truncate_insert, merge, replace
527
- loadTable = formatMultipleLoadTableName(task, taskIndex);
528
- JdbcOutputConnection con = newConnection(task, true, true);
529
- try {
530
- con.createTableIfNotExists(loadTable, insertSchema);
531
- } finally {
532
- con.close();
533
- }
534
-
535
- } else if (!mode.usesMultipleLoadTables() && mode.createAndSwapTable()) {
536
- // replace_inplace
537
- loadTable = task.getSwapTable().get();
538
-
539
- } else {
540
- // insert_direct
541
- loadTable = task.getTable();
542
- }
543
- b.prepare(loadTable, insertSchema);
544
- }
545
- });
546
-
547
- PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(),
548
- task.getBatchSize());
549
- batch = null;
550
- return output;
551
-
552
- } catch (SQLException | InterruptedException ex) {
553
- throw new RuntimeException(ex);
554
-
555
- } finally {
556
- if (batch != null) {
557
- try {
558
- batch.close();
559
- } catch (IOException | SQLException ex) {
560
- throw new RuntimeException(ex);
561
- }
562
- }
563
- }
564
- }
565
-
566
- protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
567
- TimestampFormatter timestampFormatter)
568
- {
569
- return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
570
- }
571
-
572
- protected PluginPageOutput newPluginPageOutput(PageReader reader,
573
- BatchInsert batch, List<ColumnSetter> columnSetters,
574
- int batchSize)
575
- {
576
- return new PluginPageOutput(reader, batch, columnSetters, batchSize);
577
- }
578
-
579
- public static class PluginPageOutput
580
- implements TransactionalPageOutput
581
- {
582
- protected final List<Column> columns;
583
- protected final List<ColumnSetter> columnSetters;
584
- private final PageReader pageReader;
585
- private final BatchInsert batch;
586
- private final int batchSize;
587
- private final int foraceBatchFlushSize;
588
-
589
- public PluginPageOutput(PageReader pageReader,
590
- BatchInsert batch, List<ColumnSetter> columnSetters,
591
- int batchSize)
592
- {
593
- this.pageReader = pageReader;
594
- this.batch = batch;
595
- this.columns = pageReader.getSchema().getColumns();
596
- this.columnSetters = columnSetters;
597
- this.batchSize = batchSize;
598
- this.foraceBatchFlushSize = batchSize * 2;
599
- }
600
-
601
- @Override
602
- public void add(Page page)
603
- {
604
- try {
605
- pageReader.setPage(page);
606
- while (pageReader.nextRecord()) {
607
- if (batch.getBatchWeight() > foraceBatchFlushSize) {
608
- batch.flush();
609
- }
610
- handleColumnsSetters();
611
- batch.add();
612
- }
613
- if (batch.getBatchWeight() > batchSize) {
614
- batch.flush();
615
- }
616
- } catch (IOException | SQLException ex) {
617
- throw new RuntimeException(ex);
618
- }
619
- }
620
-
621
- @Override
622
- public void finish()
623
- {
624
- try {
625
- batch.finish();
626
- } catch (IOException | SQLException ex) {
627
- throw new RuntimeException(ex);
628
- }
629
- }
630
-
631
- @Override
632
- public void close()
633
- {
634
- try {
635
- batch.close();
636
- } catch (IOException | SQLException ex) {
637
- throw new RuntimeException(ex);
638
- }
639
- }
640
-
641
- @Override
642
- public void abort()
643
- {
644
- }
645
-
646
- @Override
647
- public CommitReport commit()
648
- {
649
- return Exec.newCommitReport();
650
- }
651
-
652
- protected void handleColumnsSetters()
653
- {
654
- int size = columnSetters.size();
655
- for (int i=0; i < size; i++) {
656
- columns.get(i).visit(columnSetters.get(i));
657
- }
658
- }
659
-
660
- }
661
-
662
- public static interface IdempotentSqlRunnable
663
- {
664
- public void run() throws SQLException;
665
- }
666
-
667
- protected void withRetry(IdempotentSqlRunnable op)
668
- throws SQLException, InterruptedException
669
- {
670
- withRetry(op, "Operation failed");
671
- }
672
-
673
- protected void withRetry(final IdempotentSqlRunnable op, final String errorMessage)
674
- throws SQLException, InterruptedException
675
- {
676
- try {
677
- retryExecutor()
678
- .setRetryLimit(12)
679
- .setInitialRetryWait(1000)
680
- .setMaxRetryWait(30 * 60 * 1000)
681
- .runInterruptible(new IdempotentOperation<Void>() {
682
- public Void call() throws Exception
683
- {
684
- op.run();
685
- return null;
686
- }
687
-
688
- public void onRetry(Throwable exception, int retryCount, int retryLimit, int retryWait)
689
- {
690
- if (exception instanceof SQLException) {
691
- SQLException ex = (SQLException) exception;
692
- String sqlState = ex.getSQLState();
693
- int errorCode = ex.getErrorCode();
694
- logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
695
- errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
696
- buildExceptionMessage(exception));
697
- } else {
698
- logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
699
- errorMessage, retryCount, retryLimit, retryWait/1000,
700
- buildExceptionMessage(exception));
701
- }
702
- if (retryCount % 3 == 0) {
703
- logger.info("Error details:", exception);
704
- }
705
- }
706
-
707
- public void onGiveup(Throwable firstException, Throwable lastException)
708
- {
709
- if (firstException instanceof SQLException) {
710
- SQLException ex = (SQLException) firstException;
711
- String sqlState = ex.getSQLState();
712
- int errorCode = ex.getErrorCode();
713
- logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
714
- }
715
- }
716
-
717
- public boolean isRetryableException(Throwable exception)
718
- {
719
- //if (exception instanceof SQLException) {
720
- // SQLException ex = (SQLException) exception;
721
- // String sqlState = ex.getSQLState();
722
- // int errorCode = ex.getErrorCode();
723
- // return isRetryableSQLException(ex);
724
- //}
725
- return false; // TODO
726
- }
727
- });
728
-
729
- } catch (ExecutionException ex) {
730
- Throwable cause = ex.getCause();
731
- Throwables.propagateIfInstanceOf(cause, SQLException.class);
732
- throw Throwables.propagate(cause);
733
- }
734
- }
735
-
736
- private String buildExceptionMessage(Throwable ex) {
737
- StringBuilder sb = new StringBuilder();
738
- sb.append(ex.getMessage());
739
- if (ex.getCause() != null) {
740
- buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
741
- }
742
- return sb.toString();
743
- }
744
-
745
- private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
746
- if (!lastMessage.equals(ex.getMessage())) {
747
- // suppress same messages
748
- sb.append(" < ");
749
- sb.append(ex.getMessage());
750
- }
751
- if (ex.getCause() == null) {
752
- return;
753
- }
754
- buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
755
- }
756
- }
1
+ package org.embulk.output.jdbc;
2
+
3
+ import java.util.HashSet;
4
+ import java.util.List;
5
+ import java.util.Locale;
6
+ import java.util.Properties;
7
+ import java.util.Set;
8
+ import java.util.concurrent.ExecutionException;
9
+ import java.io.IOException;
10
+ import java.nio.file.Paths;
11
+ import java.sql.Types;
12
+ import java.sql.ResultSet;
13
+ import java.sql.DatabaseMetaData;
14
+ import java.sql.SQLException;
15
+
16
+ import org.slf4j.Logger;
17
+
18
+ import com.google.common.base.Optional;
19
+ import com.google.common.base.Throwables;
20
+ import com.google.common.collect.ImmutableList;
21
+
22
+ import org.embulk.config.CommitReport;
23
+ import org.embulk.config.Config;
24
+ import org.embulk.config.ConfigDefault;
25
+ import org.embulk.config.ConfigDiff;
26
+ import org.embulk.config.ConfigException;
27
+ import org.embulk.config.ConfigSource;
28
+ import org.embulk.config.Task;
29
+ import org.embulk.config.TaskSource;
30
+ import org.embulk.spi.Exec;
31
+ import org.embulk.spi.Column;
32
+ import org.embulk.spi.ColumnVisitor;
33
+ import org.embulk.spi.OutputPlugin;
34
+ import org.embulk.spi.PluginClassLoader;
35
+ import org.embulk.spi.Schema;
36
+ import org.embulk.spi.TransactionalPageOutput;
37
+ import org.embulk.spi.Page;
38
+ import org.embulk.spi.PageReader;
39
+ import org.embulk.spi.time.Timestamp;
40
+ import org.embulk.spi.time.TimestampFormatter;
41
+ import org.embulk.output.jdbc.setter.ColumnSetter;
42
+ import org.embulk.output.jdbc.setter.ColumnSetterFactory;
43
+ import org.embulk.output.jdbc.RetryExecutor.IdempotentOperation;
44
+
45
+ import static org.embulk.output.jdbc.RetryExecutor.retryExecutor;
46
+
47
+ public abstract class AbstractJdbcOutputPlugin
48
+ implements OutputPlugin
49
+ {
50
+ private final static Set<String> loadedJarGlobs = new HashSet<String>();
51
+
52
+ private final Logger logger = Exec.getLogger(getClass());
53
+
54
+ public interface PluginTask
55
+ extends Task
56
+ {
57
+ @Config("options")
58
+ @ConfigDefault("{}")
59
+ public Properties getOptions();
60
+
61
+ @Config("table")
62
+ public String getTable();
63
+
64
+ @Config("mode")
65
+ public String getModeConfig();
66
+
67
+ @Config("batch_size")
68
+ @ConfigDefault("16777216")
69
+ // TODO set minimum number
70
+ public int getBatchSize();
71
+
72
+ public void setMode(Mode mode);
73
+ public Mode getMode();
74
+
75
+ public JdbcSchema getLoadSchema();
76
+ public void setLoadSchema(JdbcSchema schema);
77
+
78
+ public Optional<String> getSwapTable();
79
+ public void setSwapTable(Optional<String> name);
80
+
81
+ public Optional<String> getMultipleLoadTablePrefix();
82
+ public void setMultipleLoadTablePrefix(Optional<String> prefix);
83
+ }
84
+
85
+ protected void loadDriverJar(String glob)
86
+ {
87
+ synchronized (loadedJarGlobs) {
88
+ if (!loadedJarGlobs.contains(glob)) {
89
+ // TODO match glob
90
+ PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
91
+ loader.addPath(Paths.get(glob));
92
+ loadedJarGlobs.add(glob);
93
+ }
94
+ }
95
+ }
96
+
97
+ // for subclasses to add @Config
98
+ protected Class<? extends PluginTask> getTaskClass()
99
+ {
100
+ return PluginTask.class;
101
+ }
102
+
103
+ protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
104
+
105
+ protected abstract BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException;
106
+
107
+ protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
108
+ boolean autoCommit) throws SQLException
109
+ {
110
+ return getConnector(task, retryableMetadataOperation).connect(autoCommit);
111
+ }
112
+
113
+ public enum Mode {
114
+ INSERT,
115
+ INSERT_DIRECT,
116
+ TRUNCATE_INSERT,
117
+ MERGE,
118
+ REPLACE,
119
+ REPLACE_INPLACE;
120
+ //REPLACE_PARTITIONING, // MySQL: partitioning, PostgreSQL: inheritance
121
+
122
+ public boolean isDirectWrite()
123
+ {
124
+ return this == INSERT_DIRECT;
125
+ }
126
+
127
+ public boolean isInplace()
128
+ {
129
+ return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
130
+ }
131
+
132
+ public boolean isMerge()
133
+ {
134
+ return this == MERGE;
135
+ }
136
+
137
+ public boolean usesMultipleLoadTables()
138
+ {
139
+ return !isInplace();
140
+ }
141
+
142
+ public boolean createAndSwapTable()
143
+ {
144
+ return this == REPLACE_INPLACE || this == REPLACE;
145
+ }
146
+ }
147
+
148
+ public ConfigDiff transaction(ConfigSource config,
149
+ Schema schema, int taskCount,
150
+ OutputPlugin.Control control)
151
+ {
152
+ PluginTask task = config.loadConfig(getTaskClass());
153
+
154
+ // TODO this is a temporary code. behavior will change in a future release.
155
+ switch(task.getModeConfig()) {
156
+ case "insert":
157
+ task.setMode(Mode.INSERT_DIRECT);
158
+ break;
159
+ case "replace":
160
+ task.setMode(Mode.REPLACE_INPLACE);
161
+ break;
162
+ case "merge":
163
+ task.setMode(Mode.MERGE);
164
+ break;
165
+ default:
166
+ throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
167
+ }
168
+
169
+ //switch(task.getModeConfig()) {
170
+ ////case "insert":
171
+ //// task.setMode(Mode.INSERT);
172
+ //// break;
173
+ //case "insert_direct":
174
+ // task.setMode(Mode.INSERT_DIRECT);
175
+ // break;
176
+ ////case "truncate_insert": // TODO
177
+ //// task.setMode(Mode.TRUNCATE_INSERT);
178
+ //// break;
179
+ ////case "merge": // TODO
180
+ //// task.setMode(Mode.MERGE);
181
+ //// break;
182
+ ////case "replace":
183
+ //// task.setMode(Mode.REPLACE);
184
+ //// break;
185
+ //case "replace_inplace":
186
+ // task.setMode(Mode.REPLACE_INPLACE);
187
+ // break;
188
+ //default:
189
+ // new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert_direct, replace_inplace", task.getModeConfig()));
190
+ //}
191
+
192
+ task = begin(task, schema, taskCount);
193
+ control.run(task.dump());
194
+ return commit(task, schema, taskCount);
195
+ }
196
+
197
+ public ConfigDiff resume(TaskSource taskSource,
198
+ Schema schema, int taskCount,
199
+ OutputPlugin.Control control)
200
+ {
201
+ PluginTask task = taskSource.loadTask(getTaskClass());
202
+
203
+ if (task.getMode().isInplace()) {
204
+ throw new UnsupportedOperationException("inplace mode is not resumable. You need to delete partially-loaded records from the database and restart the entire transaction.");
205
+ }
206
+
207
+ task = begin(task, schema, taskCount);
208
+ control.run(task.dump());
209
+ return commit(task, schema, taskCount);
210
+ }
211
+
212
+ protected String getTransactionUniqueName()
213
+ {
214
+ // TODO use uuid?
215
+ Timestamp t = Exec.session().getTransactionTime();
216
+ return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
217
+ }
218
+
219
+ private PluginTask begin(final PluginTask task,
220
+ final Schema schema, int taskCount)
221
+ {
222
+ try {
223
+ withRetry(new IdempotentSqlRunnable() { // no intermediate data if isDirectWrite == true
224
+ public void run() throws SQLException
225
+ {
226
+ JdbcOutputConnection con = newConnection(task, true, false);
227
+ try {
228
+ doBegin(con, task, schema);
229
+ } finally {
230
+ con.close();
231
+ }
232
+ }
233
+ });
234
+ } catch (SQLException | InterruptedException ex) {
235
+ throw new RuntimeException(ex);
236
+ }
237
+ return task;
238
+ }
239
+
240
+ private ConfigDiff commit(final PluginTask task,
241
+ Schema schema, final int taskCount)
242
+ {
243
+ if (!task.getMode().isDirectWrite()) { // no intermediate data if isDirectWrite == true
244
+ try {
245
+ withRetry(new IdempotentSqlRunnable() {
246
+ public void run() throws SQLException
247
+ {
248
+ JdbcOutputConnection con = newConnection(task, false, false);
249
+ try {
250
+ doCommit(con, task, taskCount);
251
+ } finally {
252
+ con.close();
253
+ }
254
+ }
255
+ });
256
+ } catch (SQLException | InterruptedException ex) {
257
+ throw new RuntimeException(ex);
258
+ }
259
+ }
260
+ return Exec.newConfigDiff();
261
+ }
262
+
263
+ public void cleanup(TaskSource taskSource,
264
+ Schema schema, final int taskCount,
265
+ final List<CommitReport> successCommitReports)
266
+ {
267
+ final PluginTask task = taskSource.loadTask(getTaskClass());
268
+
269
+ if (!task.getMode().isDirectWrite()) { // no intermediate data if isDirectWrite == true
270
+ try {
271
+ withRetry(new IdempotentSqlRunnable() {
272
+ public void run() throws SQLException
273
+ {
274
+ JdbcOutputConnection con = newConnection(task, true, true);
275
+ try {
276
+ doCleanup(con, task, taskCount, successCommitReports);
277
+ } finally {
278
+ con.close();
279
+ }
280
+ }
281
+ });
282
+ } catch (SQLException | InterruptedException ex) {
283
+ throw new RuntimeException(ex);
284
+ }
285
+ }
286
+ }
287
+
288
+ protected void doBegin(JdbcOutputConnection con,
289
+ PluginTask task, Schema schema) throws SQLException
290
+ {
291
+ Mode mode = task.getMode();
292
+
293
+ JdbcSchema targetTableSchema;
294
+ if (mode.createAndSwapTable()) {
295
+ // DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
296
+ // CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
297
+ // swapTableName = "xyz__0000000054d92dee1e452158_bulk_load_temp"
298
+ String swapTableName = generateSwapTableName(task);
299
+ con.dropTableIfExists(swapTableName);
300
+ con.createTableIfNotExists(swapTableName, newJdbcSchemaForNewTable(schema));
301
+ targetTableSchema = newJdbcSchemaFromExistentTable(con, swapTableName);
302
+ task.setSwapTable(Optional.of(swapTableName));
303
+ } else {
304
+ // CREATE TABLE IF NOT EXISTS xyz
305
+ con.createTableIfNotExists(task.getTable(), newJdbcSchemaForNewTable(schema));
306
+ targetTableSchema = newJdbcSchemaFromExistentTable(con, task.getTable());
307
+ task.setSwapTable(Optional.<String>absent());
308
+ }
309
+
310
+ if (mode.usesMultipleLoadTables()) {
311
+ // multipleLoadTablePrefix = "xyz__0000000054d92dee1e452158_"
312
+ // workers run:
313
+ // CREATE TABLE xyz__0000000054d92dee1e452158_%d
314
+ String multipleLoadTablePrefix = task.getTable() + "_" + getTransactionUniqueName();
315
+ task.setMultipleLoadTablePrefix(Optional.of(multipleLoadTablePrefix));
316
+ } else {
317
+ task.setMultipleLoadTablePrefix(Optional.<String>absent());
318
+ }
319
+
320
+ task.setLoadSchema(matchSchemaByColumnNames(schema, targetTableSchema));
321
+ }
322
+
323
+ protected String generateSwapTableName(PluginTask task) throws SQLException
324
+ {
325
+ return task.getTable() + "_" + getTransactionUniqueName() + "_bulk_load_temp";
326
+ }
327
+
328
+ protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
329
+ throws SQLException
330
+ {
331
+ switch (task.getMode()) {
332
+ case INSERT:
333
+ // aggregate insert into target
334
+ //con.gatherInsertTables();
335
+ throw new UnsupportedOperationException("not implemented yet"); // TODO
336
+ case INSERT_DIRECT:
337
+ // already done
338
+ break;
339
+ case TRUNCATE_INSERT:
340
+ // truncate & aggregate insert into target
341
+ throw new UnsupportedOperationException("not implemented yet");
342
+ //break;
343
+ case MERGE:
344
+ // aggregate merge into target
345
+ throw new UnsupportedOperationException("not implemented yet");
346
+ //break;
347
+ case REPLACE:
348
+ if (taskCount == 1) {
349
+ // swap table
350
+ con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
351
+ } else {
352
+ // aggregate insert into swap table & swap table
353
+ throw new UnsupportedOperationException("not implemented yet");
354
+ }
355
+ break;
356
+ case REPLACE_INPLACE:
357
+ // swap table
358
+ con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
359
+ break;
360
+ }
361
+ }
362
+
363
+ protected void doCleanup(JdbcOutputConnection con, PluginTask task, int taskCount,
364
+ List<CommitReport> successCommitReports)
365
+ throws SQLException
366
+ {
367
+ if (task.getSwapTable().isPresent()) {
368
+ con.dropTableIfExists(task.getSwapTable().get());
369
+ }
370
+ if (task.getMultipleLoadTablePrefix().isPresent()) {
371
+ for (int i=0; i < taskCount; i++) {
372
+ con.dropTableIfExists(formatMultipleLoadTableName(task, i));
373
+ }
374
+ }
375
+ }
376
+
377
+ static String formatMultipleLoadTableName(PluginTask task, int taskIndex)
378
+ {
379
+ return task.getMultipleLoadTablePrefix().get() + String.format("%04x", taskIndex);
380
+ }
381
+
382
+ protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
383
+ {
384
+ final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
385
+ for (Column c : schema.getColumns()) {
386
+ final String columnName = c.getName();
387
+ c.visit(new ColumnVisitor() {
388
+ public void booleanColumn(Column column)
389
+ {
390
+ columns.add(new JdbcColumn(
391
+ columnName, "BOOLEAN",
392
+ Types.BOOLEAN, 1, 0, false));
393
+ }
394
+
395
+ public void longColumn(Column column)
396
+ {
397
+ columns.add(new JdbcColumn(
398
+ columnName, "BIGINT",
399
+ Types.BIGINT, 22, 0, false));
400
+ }
401
+
402
+ public void doubleColumn(Column column)
403
+ {
404
+ columns.add(new JdbcColumn(
405
+ columnName, "DOUBLE PRECISION",
406
+ Types.FLOAT, 24, 0, false));
407
+ }
408
+
409
+ public void stringColumn(Column column)
410
+ {
411
+ columns.add(new JdbcColumn(
412
+ columnName, "CLOB",
413
+ Types.CLOB, 4000, 0, false)); // TODO size type param
414
+ }
415
+
416
+ public void timestampColumn(Column column)
417
+ {
418
+ columns.add(new JdbcColumn(
419
+ columnName, "TIMESTAMP",
420
+ Types.TIMESTAMP, 26, 0, false)); // size type param is from postgresql.
421
+ }
422
+ });
423
+ }
424
+ return new JdbcSchema(columns.build());
425
+ }
426
+
427
+ public JdbcSchema newJdbcSchemaFromExistentTable(JdbcOutputConnection connection,
428
+ String tableName) throws SQLException
429
+ {
430
+ DatabaseMetaData dbm = connection.getMetaData();
431
+ String escape = dbm.getSearchStringEscape();
432
+ String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
433
+
434
+ ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
435
+ ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
436
+ try {
437
+ while(rs.next()) {
438
+ primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
439
+ }
440
+ } finally {
441
+ rs.close();
442
+ }
443
+ ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
444
+
445
+ String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
446
+ ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
447
+ rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
448
+ try {
449
+ while(rs.next()) {
450
+ String columnName = rs.getString("COLUMN_NAME");
451
+ String typeName = rs.getString("TYPE_NAME");
452
+ boolean isPrimaryKey = primaryKeys.contains(columnName);
453
+ typeName = typeName.toUpperCase(Locale.ENGLISH);
454
+ int sqlType = rs.getInt("DATA_TYPE");
455
+ int colSize = rs.getInt("COLUMN_SIZE");
456
+ int decDigit = rs.getInt("DECIMAL_DIGITS");
457
+ if (rs.wasNull()) {
458
+ decDigit = -1;
459
+ }
460
+ //rs.getString("IS_NULLABLE").equals("NO") // "YES" or "" // TODO
461
+ //rs.getString("COLUMN_DEF") // or null // TODO
462
+ columns.add(new JdbcColumn(
463
+ columnName, typeName,
464
+ sqlType, colSize, decDigit, isPrimaryKey));
465
+ }
466
+ } finally {
467
+ rs.close();
468
+ }
469
+ return new JdbcSchema(columns.build());
470
+ }
471
+
472
+ private JdbcSchema matchSchemaByColumnNames(Schema inputSchema, JdbcSchema targetTableSchema)
473
+ {
474
+ ImmutableList.Builder<JdbcColumn> jdbcColumns = ImmutableList.builder();
475
+
476
+ outer : for (Column column : inputSchema.getColumns()) {
477
+ for (JdbcColumn jdbcColumn : targetTableSchema.getColumns()) {
478
+ if (jdbcColumn.getName().equals(column.getName())) {
479
+ jdbcColumns.add(jdbcColumn);
480
+ continue outer;
481
+ }
482
+ }
483
+
484
+ jdbcColumns.add(JdbcColumn.skipColumn());
485
+ }
486
+
487
+ return new JdbcSchema(jdbcColumns.build());
488
+ }
489
+
490
+ public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final int taskIndex)
491
+ {
492
+ final PluginTask task = taskSource.loadTask(getTaskClass());
493
+ final Mode mode = task.getMode();
494
+
495
+ BatchInsert batch;
496
+ try {
497
+ batch = newBatchInsert(task);
498
+ } catch (IOException | SQLException ex) {
499
+ throw new RuntimeException(ex);
500
+ }
501
+ try {
502
+ PageReader reader = new PageReader(schema);
503
+ ColumnSetterFactory factory = newColumnSetterFactory(batch, reader, null); // TODO TimestampFormatter
504
+
505
+ JdbcSchema loadSchema = task.getLoadSchema();
506
+
507
+ ImmutableList.Builder<JdbcColumn> insertColumns = ImmutableList.builder();
508
+ ImmutableList.Builder<ColumnSetter> columnSetters = ImmutableList.builder();
509
+ for (JdbcColumn c : loadSchema.getColumns()) {
510
+ if (c.isSkipColumn()) {
511
+ columnSetters.add(factory.newSkipColumnSetter());
512
+ } else {
513
+ columnSetters.add(factory.newColumnSetter(c));
514
+ insertColumns.add(c);
515
+ }
516
+ }
517
+ final JdbcSchema insertSchema = new JdbcSchema(insertColumns.build());
518
+
519
+ final BatchInsert b = batch;
520
+ withRetry(new IdempotentSqlRunnable() {
521
+ public void run() throws SQLException
522
+ {
523
+ String loadTable;
524
+ boolean createTable;
525
+ if (mode.usesMultipleLoadTables()) {
526
+ // insert, truncate_insert, merge, replace
527
+ loadTable = formatMultipleLoadTableName(task, taskIndex);
528
+ JdbcOutputConnection con = newConnection(task, true, true);
529
+ try {
530
+ con.createTableIfNotExists(loadTable, insertSchema);
531
+ } finally {
532
+ con.close();
533
+ }
534
+
535
+ } else if (!mode.usesMultipleLoadTables() && mode.createAndSwapTable()) {
536
+ // replace_inplace
537
+ loadTable = task.getSwapTable().get();
538
+
539
+ } else {
540
+ // insert_direct
541
+ loadTable = task.getTable();
542
+ }
543
+ b.prepare(loadTable, insertSchema);
544
+ }
545
+ });
546
+
547
+ PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(), task);
548
+ batch = null;
549
+ return output;
550
+
551
+ } catch (SQLException | InterruptedException ex) {
552
+ throw new RuntimeException(ex);
553
+
554
+ } finally {
555
+ if (batch != null) {
556
+ try {
557
+ batch.close();
558
+ } catch (IOException | SQLException ex) {
559
+ throw new RuntimeException(ex);
560
+ }
561
+ }
562
+ }
563
+ }
564
+
565
+ protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
566
+ TimestampFormatter timestampFormatter)
567
+ {
568
+ return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
569
+ }
570
+
571
+ protected PluginPageOutput newPluginPageOutput(PageReader reader,
572
+ BatchInsert batch, List<ColumnSetter> columnSetters,
573
+ PluginTask task)
574
+ {
575
+ return new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
576
+ }
577
+
578
+ public static class PluginPageOutput
579
+ implements TransactionalPageOutput
580
+ {
581
+ protected final List<Column> columns;
582
+ protected final List<ColumnSetter> columnSetters;
583
+ private final PageReader pageReader;
584
+ private final BatchInsert batch;
585
+ private final int batchSize;
586
+ private final int foraceBatchFlushSize;
587
+
588
+ public PluginPageOutput(PageReader pageReader,
589
+ BatchInsert batch, List<ColumnSetter> columnSetters,
590
+ int batchSize)
591
+ {
592
+ this.pageReader = pageReader;
593
+ this.batch = batch;
594
+ this.columns = pageReader.getSchema().getColumns();
595
+ this.columnSetters = columnSetters;
596
+ this.batchSize = batchSize;
597
+ this.foraceBatchFlushSize = batchSize * 2;
598
+ }
599
+
600
+ @Override
601
+ public void add(Page page)
602
+ {
603
+ try {
604
+ pageReader.setPage(page);
605
+ while (pageReader.nextRecord()) {
606
+ if (batch.getBatchWeight() > foraceBatchFlushSize) {
607
+ batch.flush();
608
+ }
609
+ handleColumnsSetters();
610
+ batch.add();
611
+ }
612
+ if (batch.getBatchWeight() > batchSize) {
613
+ batch.flush();
614
+ }
615
+ } catch (IOException | SQLException ex) {
616
+ throw new RuntimeException(ex);
617
+ }
618
+ }
619
+
620
+ @Override
621
+ public void finish()
622
+ {
623
+ try {
624
+ batch.finish();
625
+ } catch (IOException | SQLException ex) {
626
+ throw new RuntimeException(ex);
627
+ }
628
+ }
629
+
630
+ @Override
631
+ public void close()
632
+ {
633
+ try {
634
+ batch.close();
635
+ } catch (IOException | SQLException ex) {
636
+ throw new RuntimeException(ex);
637
+ }
638
+ }
639
+
640
+ @Override
641
+ public void abort()
642
+ {
643
+ }
644
+
645
+ @Override
646
+ public CommitReport commit()
647
+ {
648
+ return Exec.newCommitReport();
649
+ }
650
+
651
+ protected void handleColumnsSetters()
652
+ {
653
+ int size = columnSetters.size();
654
+ for (int i=0; i < size; i++) {
655
+ columns.get(i).visit(columnSetters.get(i));
656
+ }
657
+ }
658
+
659
+ }
660
+
661
+ public static interface IdempotentSqlRunnable
662
+ {
663
+ public void run() throws SQLException;
664
+ }
665
+
666
+ protected void withRetry(IdempotentSqlRunnable op)
667
+ throws SQLException, InterruptedException
668
+ {
669
+ withRetry(op, "Operation failed");
670
+ }
671
+
672
+ protected void withRetry(final IdempotentSqlRunnable op, final String errorMessage)
673
+ throws SQLException, InterruptedException
674
+ {
675
+ try {
676
+ retryExecutor()
677
+ .setRetryLimit(12)
678
+ .setInitialRetryWait(1000)
679
+ .setMaxRetryWait(30 * 60 * 1000)
680
+ .runInterruptible(new IdempotentOperation<Void>() {
681
+ public Void call() throws Exception
682
+ {
683
+ op.run();
684
+ return null;
685
+ }
686
+
687
+ public void onRetry(Throwable exception, int retryCount, int retryLimit, int retryWait)
688
+ {
689
+ if (exception instanceof SQLException) {
690
+ SQLException ex = (SQLException) exception;
691
+ String sqlState = ex.getSQLState();
692
+ int errorCode = ex.getErrorCode();
693
+ logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
694
+ errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
695
+ buildExceptionMessage(exception));
696
+ } else {
697
+ logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
698
+ errorMessage, retryCount, retryLimit, retryWait/1000,
699
+ buildExceptionMessage(exception));
700
+ }
701
+ if (retryCount % 3 == 0) {
702
+ logger.info("Error details:", exception);
703
+ }
704
+ }
705
+
706
+ public void onGiveup(Throwable firstException, Throwable lastException)
707
+ {
708
+ if (firstException instanceof SQLException) {
709
+ SQLException ex = (SQLException) firstException;
710
+ String sqlState = ex.getSQLState();
711
+ int errorCode = ex.getErrorCode();
712
+ logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
713
+ }
714
+ }
715
+
716
+ public boolean isRetryableException(Throwable exception)
717
+ {
718
+ //if (exception instanceof SQLException) {
719
+ // SQLException ex = (SQLException) exception;
720
+ // String sqlState = ex.getSQLState();
721
+ // int errorCode = ex.getErrorCode();
722
+ // return isRetryableSQLException(ex);
723
+ //}
724
+ return false; // TODO
725
+ }
726
+ });
727
+
728
+ } catch (ExecutionException ex) {
729
+ Throwable cause = ex.getCause();
730
+ Throwables.propagateIfInstanceOf(cause, SQLException.class);
731
+ throw Throwables.propagate(cause);
732
+ }
733
+ }
734
+
735
+ private String buildExceptionMessage(Throwable ex) {
736
+ StringBuilder sb = new StringBuilder();
737
+ sb.append(ex.getMessage());
738
+ if (ex.getCause() != null) {
739
+ buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
740
+ }
741
+ return sb.toString();
742
+ }
743
+
744
+ private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
745
+ if (!lastMessage.equals(ex.getMessage())) {
746
+ // suppress same messages
747
+ sb.append(" < ");
748
+ sb.append(ex.getMessage());
749
+ }
750
+ if (ex.getCause() == null) {
751
+ return;
752
+ }
753
+ buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
754
+ }
755
+ }