embulk-output-jdbc 0.7.8 → 0.7.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-output-jdbc-0.7.9.jar +0 -0
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +109 -38
- data/src/main/java/org/embulk/output/jdbc/BatchInsert.java +1 -1
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +82 -31
- data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +2 -2
- data/src/main/java/org/embulk/output/jdbc/TableIdentifier.java +54 -0
- metadata +4 -3
- data/classpath/embulk-output-jdbc-0.7.8.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 48a1142e7d456697b9e0d0c005cde55dec3d7084
|
4
|
+
data.tar.gz: e7616b21132acca2bb8aeddbb4eb59f4a5ee5f8a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d45ff98b8f76f3c0c293a583e8a29a0a1bfb78c5d19f29dd217e129819f968280167dffcb973fd5772d6dae154f52ee1c03766e6a29e471a9fd757d8e337fa45
|
7
|
+
data.tar.gz: 3a45ce034e613ba50a2361d5ccb6de5a9dc8588de7670ca4fa29bbc17ae2a495c6e24bfc41390f2b894d96514104585d2d5a5adc6744e8fdd7819b85954e243a
|
Binary file
|
@@ -15,6 +15,7 @@ import java.sql.DatabaseMetaData;
|
|
15
15
|
import java.sql.SQLException;
|
16
16
|
|
17
17
|
import org.embulk.spi.util.RetryExecutor;
|
18
|
+
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
18
19
|
import org.slf4j.Logger;
|
19
20
|
import org.joda.time.DateTimeZone;
|
20
21
|
|
@@ -116,8 +117,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
116
117
|
@ConfigDefault("null")
|
117
118
|
public Optional<String> getAfterLoad();
|
118
119
|
|
119
|
-
public void setActualTable(
|
120
|
-
public
|
120
|
+
public void setActualTable(TableIdentifier actualTable);
|
121
|
+
public TableIdentifier getActualTable();
|
121
122
|
|
122
123
|
public void setMergeKeys(Optional<List<String>> keys);
|
123
124
|
|
@@ -130,8 +131,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
130
131
|
public JdbcSchema getTargetTableSchema();
|
131
132
|
public void setTargetTableSchema(JdbcSchema schema);
|
132
133
|
|
133
|
-
public Optional<List<
|
134
|
-
public void setIntermediateTables(Optional<List<
|
134
|
+
public Optional<List<TableIdentifier>> getIntermediateTables();
|
135
|
+
public void setIntermediateTables(Optional<List<TableIdentifier>> names);
|
135
136
|
}
|
136
137
|
|
137
138
|
public static enum LengthSemantics
|
@@ -461,8 +462,9 @@ public abstract class AbstractJdbcOutputPlugin
|
|
461
462
|
throw new ConfigException(String.format("%s mode does not support 'before_load' option.", mode));
|
462
463
|
}
|
463
464
|
|
465
|
+
String actualTable;
|
464
466
|
if (con.tableExists(task.getTable())) {
|
465
|
-
task.
|
467
|
+
actualTable = task.getTable();
|
466
468
|
} else {
|
467
469
|
String upperTable = task.getTable().toUpperCase();
|
468
470
|
String lowerTable = task.getTable().toLowerCase();
|
@@ -471,16 +473,17 @@ public abstract class AbstractJdbcOutputPlugin
|
|
471
473
|
throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
|
472
474
|
task.getTable(), upperTable, lowerTable));
|
473
475
|
} else {
|
474
|
-
|
476
|
+
actualTable = upperTable;
|
475
477
|
}
|
476
478
|
} else {
|
477
479
|
if (con.tableExists(lowerTable)) {
|
478
|
-
|
480
|
+
actualTable = lowerTable;
|
479
481
|
} else {
|
480
|
-
task.
|
482
|
+
actualTable = task.getTable();
|
481
483
|
}
|
482
484
|
}
|
483
485
|
}
|
486
|
+
task.setActualTable(new TableIdentifier(null, con.getSchemaName(), actualTable));
|
484
487
|
|
485
488
|
Optional<JdbcSchema> initialTargetTableSchema =
|
486
489
|
mode.ignoreTargetTableSchema() ?
|
@@ -499,29 +502,11 @@ public abstract class AbstractJdbcOutputPlugin
|
|
499
502
|
|
500
503
|
// create intermediate tables
|
501
504
|
if (!mode.isDirectModify()) {
|
502
|
-
// direct modify mode doesn't need intermediate tables.
|
503
|
-
ImmutableList.Builder<String> intermTableNames = ImmutableList.builder();
|
504
|
-
if (mode.tempTablePerTask()) {
|
505
|
-
String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable(), con, 3,
|
506
|
-
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
507
|
-
for (int i = 0; i < taskCount; i++) {
|
508
|
-
intermTableNames.add(namePrefix + String.format("%03d", i));
|
509
|
-
}
|
510
|
-
} else {
|
511
|
-
String name = generateIntermediateTableNamePrefix(task.getActualTable(), con, 0,
|
512
|
-
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
513
|
-
intermTableNames.add(name);
|
514
|
-
}
|
515
505
|
// create the intermediate tables here
|
516
|
-
task.setIntermediateTables(Optional.<List<
|
517
|
-
for (String name : task.getIntermediateTables().get()) {
|
518
|
-
// DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
|
519
|
-
con.dropTableIfExists(name);
|
520
|
-
// CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
|
521
|
-
con.createTableIfNotExists(name, newTableSchema);
|
522
|
-
}
|
506
|
+
task.setIntermediateTables(Optional.<List<TableIdentifier>>of(createIntermediateTables(con, task, taskCount, newTableSchema)));
|
523
507
|
} else {
|
524
|
-
|
508
|
+
// direct modify mode doesn't need intermediate tables.
|
509
|
+
task.setIntermediateTables(Optional.<List<TableIdentifier>>absent());
|
525
510
|
if (task.getBeforeLoad().isPresent()) {
|
526
511
|
con.executeSql(task.getBeforeLoad().get());
|
527
512
|
}
|
@@ -533,7 +518,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
533
518
|
targetTableSchema = initialTargetTableSchema.get();
|
534
519
|
task.setNewTableSchema(Optional.<JdbcSchema>absent());
|
535
520
|
} else if (task.getIntermediateTables().isPresent() && !task.getIntermediateTables().get().isEmpty()) {
|
536
|
-
|
521
|
+
TableIdentifier firstItermTable = task.getIntermediateTables().get().get(0);
|
537
522
|
targetTableSchema = newJdbcSchemaFromTableIfExists(con, firstItermTable).get();
|
538
523
|
task.setNewTableSchema(Optional.of(newTableSchema));
|
539
524
|
} else {
|
@@ -591,6 +576,91 @@ public abstract class AbstractJdbcOutputPlugin
|
|
591
576
|
return new ColumnSetterFactory(batch, defaultTimeZone);
|
592
577
|
}
|
593
578
|
|
579
|
+
protected TableIdentifier buildIntermediateTableId(final JdbcOutputConnection con, PluginTask task, String tableName)
|
580
|
+
{
|
581
|
+
return new TableIdentifier(null, con.getSchemaName(), tableName);
|
582
|
+
}
|
583
|
+
|
584
|
+
private List<TableIdentifier> createIntermediateTables(final JdbcOutputConnection con,
|
585
|
+
final PluginTask task, final int taskCount, final JdbcSchema newTableSchema) throws SQLException
|
586
|
+
{
|
587
|
+
try {
|
588
|
+
return buildRetryExecutor(task).run(new Retryable<List<TableIdentifier>>() {
|
589
|
+
private TableIdentifier table;
|
590
|
+
private ImmutableList.Builder<TableIdentifier> intermTables;
|
591
|
+
|
592
|
+
@Override
|
593
|
+
public List<TableIdentifier> call() throws Exception
|
594
|
+
{
|
595
|
+
intermTables = ImmutableList.builder();
|
596
|
+
if (task.getMode().tempTablePerTask()) {
|
597
|
+
String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 3,
|
598
|
+
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
599
|
+
for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
|
600
|
+
String tableName = namePrefix + String.format("%03d", taskIndex);
|
601
|
+
table = buildIntermediateTableId(con, task, tableName);
|
602
|
+
// if table already exists, SQLException will be thrown
|
603
|
+
con.createTable(table, newTableSchema);
|
604
|
+
intermTables.add(table);
|
605
|
+
}
|
606
|
+
} else {
|
607
|
+
String tableName = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 0,
|
608
|
+
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
609
|
+
table = buildIntermediateTableId(con, task, tableName);
|
610
|
+
con.createTable(table, newTableSchema);
|
611
|
+
intermTables.add(table);
|
612
|
+
}
|
613
|
+
return intermTables.build();
|
614
|
+
}
|
615
|
+
|
616
|
+
@Override
|
617
|
+
public boolean isRetryableException(Exception exception)
|
618
|
+
{
|
619
|
+
if (exception instanceof SQLException) {
|
620
|
+
try {
|
621
|
+
// true means that creating table failed because the table already exists.
|
622
|
+
return con.tableExists(table);
|
623
|
+
} catch (SQLException e) {
|
624
|
+
}
|
625
|
+
}
|
626
|
+
return false;
|
627
|
+
}
|
628
|
+
|
629
|
+
@Override
|
630
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
631
|
+
throws RetryGiveupException
|
632
|
+
{
|
633
|
+
logger.info("Try to create intermediate tables again because already exist");
|
634
|
+
try {
|
635
|
+
dropTables();
|
636
|
+
} catch (SQLException e) {
|
637
|
+
throw new RetryGiveupException(e);
|
638
|
+
}
|
639
|
+
}
|
640
|
+
|
641
|
+
@Override
|
642
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
643
|
+
throws RetryGiveupException
|
644
|
+
{
|
645
|
+
try {
|
646
|
+
dropTables();
|
647
|
+
} catch (SQLException e) {
|
648
|
+
logger.warn("Cannot delete intermediate table", e);
|
649
|
+
}
|
650
|
+
}
|
651
|
+
|
652
|
+
private void dropTables() throws SQLException
|
653
|
+
{
|
654
|
+
for (TableIdentifier table : intermTables.build()) {
|
655
|
+
con.dropTableIfExists(table);
|
656
|
+
}
|
657
|
+
}
|
658
|
+
});
|
659
|
+
} catch (RetryGiveupException e) {
|
660
|
+
throw new RuntimeException(e);
|
661
|
+
}
|
662
|
+
}
|
663
|
+
|
594
664
|
protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con,
|
595
665
|
int suffixLength, int maxLength, LengthSemantics lengthSemantics) throws SQLException
|
596
666
|
{
|
@@ -729,7 +799,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
729
799
|
throws SQLException
|
730
800
|
{
|
731
801
|
if (task.getIntermediateTables().isPresent()) {
|
732
|
-
for (
|
802
|
+
for (TableIdentifier intermTable : task.getIntermediateTables().get()) {
|
733
803
|
con.dropTableIfExists(intermTable);
|
734
804
|
}
|
735
805
|
}
|
@@ -788,9 +858,9 @@ public abstract class AbstractJdbcOutputPlugin
|
|
788
858
|
}
|
789
859
|
|
790
860
|
public Optional<JdbcSchema> newJdbcSchemaFromTableIfExists(JdbcOutputConnection connection,
|
791
|
-
|
861
|
+
TableIdentifier table) throws SQLException
|
792
862
|
{
|
793
|
-
if (!connection.tableExists(
|
863
|
+
if (!connection.tableExists(table)) {
|
794
864
|
// DatabaseMetaData.getPrimaryKeys fails if table does not exist
|
795
865
|
return Optional.absent();
|
796
866
|
}
|
@@ -798,7 +868,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
798
868
|
DatabaseMetaData dbm = connection.getMetaData();
|
799
869
|
String escape = dbm.getSearchStringEscape();
|
800
870
|
|
801
|
-
ResultSet rs = dbm.getPrimaryKeys(
|
871
|
+
ResultSet rs = dbm.getPrimaryKeys(table.getDatabase(), table.getSchemaName(), table.getTableName());
|
802
872
|
ImmutableSet.Builder<String> primaryKeysBuilder = ImmutableSet.builder();
|
803
873
|
try {
|
804
874
|
while(rs.next()) {
|
@@ -810,9 +880,10 @@ public abstract class AbstractJdbcOutputPlugin
|
|
810
880
|
ImmutableSet<String> primaryKeys = primaryKeysBuilder.build();
|
811
881
|
|
812
882
|
ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
|
813
|
-
rs = dbm.getColumns(
|
814
|
-
JdbcUtils.escapeSearchString(
|
815
|
-
JdbcUtils.escapeSearchString(
|
883
|
+
rs = dbm.getColumns(
|
884
|
+
JdbcUtils.escapeSearchString(table.getDatabase(), escape),
|
885
|
+
JdbcUtils.escapeSearchString(table.getSchemaName(), escape),
|
886
|
+
JdbcUtils.escapeSearchString(table.getTableName(), escape),
|
816
887
|
null);
|
817
888
|
try {
|
818
889
|
while (rs.next()) {
|
@@ -887,7 +958,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
887
958
|
}
|
888
959
|
|
889
960
|
// configure BatchInsert -> an intermediate table (!isDirectModify) or the target table (isDirectModify)
|
890
|
-
|
961
|
+
TableIdentifier destTable;
|
891
962
|
if (mode.tempTablePerTask()) {
|
892
963
|
destTable = task.getIntermediateTables().get().get(taskIndex);
|
893
964
|
} else if (mode.isDirectModify()) {
|
@@ -8,7 +8,7 @@ import org.embulk.spi.time.Timestamp;
|
|
8
8
|
|
9
9
|
public interface BatchInsert
|
10
10
|
{
|
11
|
-
public void prepare(
|
11
|
+
public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException;
|
12
12
|
|
13
13
|
public int getBatchWeight();
|
14
14
|
|
@@ -11,7 +11,9 @@ import java.sql.SQLException;
|
|
11
11
|
import java.sql.Statement;
|
12
12
|
|
13
13
|
import org.slf4j.Logger;
|
14
|
+
|
14
15
|
import com.google.common.base.Optional;
|
16
|
+
|
15
17
|
import org.embulk.spi.Exec;
|
16
18
|
|
17
19
|
public class JdbcOutputConnection
|
@@ -70,18 +72,23 @@ public class JdbcOutputConnection
|
|
70
72
|
}
|
71
73
|
}
|
72
74
|
|
73
|
-
public boolean tableExists(
|
75
|
+
public boolean tableExists(TableIdentifier table) throws SQLException
|
74
76
|
{
|
75
|
-
try (ResultSet rs = connection.getMetaData().getTables(
|
77
|
+
try (ResultSet rs = connection.getMetaData().getTables(table.getDatabase(), table.getSchemaName(), table.getTableName(), null)) {
|
76
78
|
return rs.next();
|
77
79
|
}
|
78
80
|
}
|
79
81
|
|
80
|
-
public
|
82
|
+
public boolean tableExists(String tableName) throws SQLException
|
83
|
+
{
|
84
|
+
return tableExists(new TableIdentifier(null, schemaName, tableName));
|
85
|
+
}
|
86
|
+
|
87
|
+
public void dropTableIfExists(TableIdentifier table) throws SQLException
|
81
88
|
{
|
82
89
|
Statement stmt = connection.createStatement();
|
83
90
|
try {
|
84
|
-
dropTableIfExists(stmt,
|
91
|
+
dropTableIfExists(stmt, table);
|
85
92
|
commitIfNecessary(connection);
|
86
93
|
} catch (SQLException ex) {
|
87
94
|
throw safeRollback(connection, ex);
|
@@ -90,17 +97,17 @@ public class JdbcOutputConnection
|
|
90
97
|
}
|
91
98
|
}
|
92
99
|
|
93
|
-
protected void dropTableIfExists(Statement stmt,
|
100
|
+
protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
|
94
101
|
{
|
95
|
-
String sql = String.format("DROP TABLE IF EXISTS %s",
|
102
|
+
String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
|
96
103
|
executeUpdate(stmt, sql);
|
97
104
|
}
|
98
105
|
|
99
|
-
public void dropTable(
|
106
|
+
public void dropTable(TableIdentifier table) throws SQLException
|
100
107
|
{
|
101
108
|
Statement stmt = connection.createStatement();
|
102
109
|
try {
|
103
|
-
dropTable(stmt,
|
110
|
+
dropTable(stmt, table);
|
104
111
|
commitIfNecessary(connection);
|
105
112
|
} catch (SQLException ex) {
|
106
113
|
throw safeRollback(connection, ex);
|
@@ -109,17 +116,17 @@ public class JdbcOutputConnection
|
|
109
116
|
}
|
110
117
|
}
|
111
118
|
|
112
|
-
protected void dropTable(Statement stmt,
|
119
|
+
protected void dropTable(Statement stmt, TableIdentifier table) throws SQLException
|
113
120
|
{
|
114
|
-
String sql = String.format("DROP TABLE %s",
|
121
|
+
String sql = String.format("DROP TABLE %s", quoteTableIdentifier(table));
|
115
122
|
executeUpdate(stmt, sql);
|
116
123
|
}
|
117
124
|
|
118
|
-
public void createTableIfNotExists(
|
125
|
+
public void createTableIfNotExists(TableIdentifier targetTable, JdbcSchema schema) throws SQLException
|
119
126
|
{
|
120
127
|
Statement stmt = connection.createStatement();
|
121
128
|
try {
|
122
|
-
String sql = buildCreateTableIfNotExistsSql(
|
129
|
+
String sql = buildCreateTableIfNotExistsSql(targetTable, schema);
|
123
130
|
executeUpdate(stmt, sql);
|
124
131
|
commitIfNecessary(connection);
|
125
132
|
} catch (SQLException ex) {
|
@@ -129,12 +136,36 @@ public class JdbcOutputConnection
|
|
129
136
|
}
|
130
137
|
}
|
131
138
|
|
132
|
-
protected String buildCreateTableIfNotExistsSql(
|
139
|
+
protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema)
|
133
140
|
{
|
134
141
|
StringBuilder sb = new StringBuilder();
|
135
142
|
|
136
143
|
sb.append("CREATE TABLE IF NOT EXISTS ");
|
137
|
-
|
144
|
+
quoteTableIdentifier(sb, table);
|
145
|
+
sb.append(buildCreateTableSchemaSql(schema));
|
146
|
+
return sb.toString();
|
147
|
+
}
|
148
|
+
|
149
|
+
public void createTable(TableIdentifier table, JdbcSchema schema) throws SQLException
|
150
|
+
{
|
151
|
+
Statement stmt = connection.createStatement();
|
152
|
+
try {
|
153
|
+
String sql = buildCreateTableSql(table, schema);
|
154
|
+
executeUpdate(stmt, sql);
|
155
|
+
commitIfNecessary(connection);
|
156
|
+
} catch (SQLException ex) {
|
157
|
+
throw safeRollback(connection, ex);
|
158
|
+
} finally {
|
159
|
+
stmt.close();
|
160
|
+
}
|
161
|
+
}
|
162
|
+
|
163
|
+
protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema)
|
164
|
+
{
|
165
|
+
StringBuilder sb = new StringBuilder();
|
166
|
+
|
167
|
+
sb.append("CREATE TABLE ");
|
168
|
+
quoteTableIdentifier(sb, table);
|
138
169
|
sb.append(buildCreateTableSchemaSql(schema));
|
139
170
|
return sb.toString();
|
140
171
|
}
|
@@ -156,13 +187,13 @@ public class JdbcOutputConnection
|
|
156
187
|
return sb.toString();
|
157
188
|
}
|
158
189
|
|
159
|
-
protected String buildRenameTableSql(
|
190
|
+
protected String buildRenameTableSql(TableIdentifier fromTable, TableIdentifier toTable)
|
160
191
|
{
|
161
192
|
StringBuilder sb = new StringBuilder();
|
162
193
|
sb.append("ALTER TABLE ");
|
163
|
-
|
194
|
+
quoteTableIdentifier(sb, fromTable);
|
164
195
|
sb.append(" RENAME TO ");
|
165
|
-
|
196
|
+
quoteTableIdentifier(sb, toTable);
|
166
197
|
return sb.toString();
|
167
198
|
}
|
168
199
|
|
@@ -241,7 +272,7 @@ public class JdbcOutputConnection
|
|
241
272
|
return ColumnDeclareType.SIMPLE;
|
242
273
|
}
|
243
274
|
|
244
|
-
public PreparedStatement prepareBatchInsertStatement(
|
275
|
+
public PreparedStatement prepareBatchInsertStatement(TableIdentifier toTable, JdbcSchema toTableSchema, Optional<MergeConfig> mergeConfig) throws SQLException
|
245
276
|
{
|
246
277
|
String sql;
|
247
278
|
if (mergeConfig.isPresent()) {
|
@@ -253,12 +284,12 @@ public class JdbcOutputConnection
|
|
253
284
|
return connection.prepareStatement(sql);
|
254
285
|
}
|
255
286
|
|
256
|
-
protected String buildPreparedInsertSql(
|
287
|
+
protected String buildPreparedInsertSql(TableIdentifier toTable, JdbcSchema toTableSchema) throws SQLException
|
257
288
|
{
|
258
289
|
StringBuilder sb = new StringBuilder();
|
259
290
|
|
260
291
|
sb.append("INSERT INTO ");
|
261
|
-
|
292
|
+
quoteTableIdentifier(sb, toTable);
|
262
293
|
|
263
294
|
sb.append(" (");
|
264
295
|
for (int i=0; i < toTableSchema.getCount(); i++) {
|
@@ -275,7 +306,7 @@ public class JdbcOutputConnection
|
|
275
306
|
return sb.toString();
|
276
307
|
}
|
277
308
|
|
278
|
-
protected String buildPreparedMergeSql(
|
309
|
+
protected String buildPreparedMergeSql(TableIdentifier toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
|
279
310
|
{
|
280
311
|
throw new UnsupportedOperationException("not implemented");
|
281
312
|
}
|
@@ -293,7 +324,7 @@ public class JdbcOutputConnection
|
|
293
324
|
}
|
294
325
|
}
|
295
326
|
|
296
|
-
protected void collectInsert(List<
|
327
|
+
protected void collectInsert(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable,
|
297
328
|
boolean truncateDestinationFirst, Optional<String> preSql, Optional<String> postSql) throws SQLException
|
298
329
|
{
|
299
330
|
if (fromTables.isEmpty()) {
|
@@ -326,22 +357,22 @@ public class JdbcOutputConnection
|
|
326
357
|
}
|
327
358
|
}
|
328
359
|
|
329
|
-
protected String buildTruncateSql(
|
360
|
+
protected String buildTruncateSql(TableIdentifier table)
|
330
361
|
{
|
331
362
|
StringBuilder sb = new StringBuilder();
|
332
363
|
|
333
364
|
sb.append("DELETE FROM ");
|
334
|
-
|
365
|
+
quoteTableIdentifier(sb, table);
|
335
366
|
|
336
367
|
return sb.toString();
|
337
368
|
}
|
338
369
|
|
339
|
-
protected String buildCollectInsertSql(List<
|
370
|
+
protected String buildCollectInsertSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable)
|
340
371
|
{
|
341
372
|
StringBuilder sb = new StringBuilder();
|
342
373
|
|
343
374
|
sb.append("INSERT INTO ");
|
344
|
-
|
375
|
+
quoteTableIdentifier(sb, toTable);
|
345
376
|
sb.append(" (");
|
346
377
|
for (int i=0; i < schema.getCount(); i++) {
|
347
378
|
if (i != 0) { sb.append(", "); }
|
@@ -356,13 +387,13 @@ public class JdbcOutputConnection
|
|
356
387
|
quoteIdentifierString(sb, schema.getColumnName(j));
|
357
388
|
}
|
358
389
|
sb.append(" FROM ");
|
359
|
-
|
390
|
+
quoteTableIdentifier(sb, fromTables.get(i));
|
360
391
|
}
|
361
392
|
|
362
393
|
return sb.toString();
|
363
394
|
}
|
364
395
|
|
365
|
-
protected void collectMerge(List<
|
396
|
+
protected void collectMerge(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig,
|
366
397
|
Optional<String> preSql, Optional<String> postSql) throws SQLException
|
367
398
|
{
|
368
399
|
if (fromTables.isEmpty()) {
|
@@ -390,12 +421,12 @@ public class JdbcOutputConnection
|
|
390
421
|
}
|
391
422
|
}
|
392
423
|
|
393
|
-
protected String buildCollectMergeSql(List<
|
424
|
+
protected String buildCollectMergeSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig) throws SQLException
|
394
425
|
{
|
395
426
|
throw new UnsupportedOperationException("not implemented");
|
396
427
|
}
|
397
428
|
|
398
|
-
public void replaceTable(
|
429
|
+
public void replaceTable(TableIdentifier fromTable, JdbcSchema schema, TableIdentifier toTable, Optional<String> postSql) throws SQLException
|
399
430
|
{
|
400
431
|
Statement stmt = connection.createStatement();
|
401
432
|
try {
|
@@ -415,9 +446,29 @@ public class JdbcOutputConnection
|
|
415
446
|
}
|
416
447
|
}
|
417
448
|
|
449
|
+
protected String quoteTableIdentifier(TableIdentifier table)
|
450
|
+
{
|
451
|
+
StringBuilder sb = new StringBuilder();
|
452
|
+
if (table.getDatabase() != null) {
|
453
|
+
sb.append(quoteIdentifierString(table.getDatabase(), identifierQuoteString));
|
454
|
+
sb.append(".");
|
455
|
+
}
|
456
|
+
if (table.getSchemaName() != null) {
|
457
|
+
sb.append(quoteIdentifierString(table.getSchemaName(), identifierQuoteString));
|
458
|
+
sb.append(".");
|
459
|
+
}
|
460
|
+
sb.append(quoteIdentifierString(table.getTableName(), identifierQuoteString));
|
461
|
+
return sb.toString();
|
462
|
+
}
|
463
|
+
|
464
|
+
protected void quoteTableIdentifier(StringBuilder sb, TableIdentifier table)
|
465
|
+
{
|
466
|
+
sb.append(quoteTableIdentifier(table));
|
467
|
+
}
|
468
|
+
|
418
469
|
protected void quoteIdentifierString(StringBuilder sb, String str)
|
419
470
|
{
|
420
|
-
sb.append(quoteIdentifierString(str
|
471
|
+
sb.append(quoteIdentifierString(str));
|
421
472
|
}
|
422
473
|
|
423
474
|
protected String quoteIdentifierString(String str)
|
@@ -33,7 +33,7 @@ public class StandardBatchInsert
|
|
33
33
|
this.mergeConfig = mergeConfig;
|
34
34
|
}
|
35
35
|
|
36
|
-
public void prepare(
|
36
|
+
public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
|
37
37
|
{
|
38
38
|
this.connection = connector.connect(true);
|
39
39
|
this.index = 1; // PreparedStatement index begings from 1
|
@@ -43,7 +43,7 @@ public class StandardBatchInsert
|
|
43
43
|
batch.clearBatch();
|
44
44
|
}
|
45
45
|
|
46
|
-
protected PreparedStatement prepareStatement(
|
46
|
+
protected PreparedStatement prepareStatement(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
|
47
47
|
{
|
48
48
|
return connection.prepareBatchInsertStatement(loadTable, insertSchema, mergeConfig);
|
49
49
|
}
|
@@ -0,0 +1,54 @@
|
|
1
|
+
package org.embulk.output.jdbc;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonProperty;
|
4
|
+
|
5
|
+
|
6
|
+
public class TableIdentifier
|
7
|
+
{
|
8
|
+
private String database;
|
9
|
+
private String schemaName;
|
10
|
+
private String tableName;
|
11
|
+
|
12
|
+
public TableIdentifier(String database, String shcemaName, String tableName)
|
13
|
+
{
|
14
|
+
this.database = database;
|
15
|
+
this.schemaName = shcemaName;
|
16
|
+
this.tableName = tableName;
|
17
|
+
}
|
18
|
+
|
19
|
+
public TableIdentifier()
|
20
|
+
{
|
21
|
+
}
|
22
|
+
|
23
|
+
@JsonProperty
|
24
|
+
public String getDatabase() {
|
25
|
+
return database;
|
26
|
+
}
|
27
|
+
|
28
|
+
@JsonProperty
|
29
|
+
public void setDatabase(String database) {
|
30
|
+
this.database = database;
|
31
|
+
}
|
32
|
+
|
33
|
+
@JsonProperty
|
34
|
+
public String getSchemaName()
|
35
|
+
{
|
36
|
+
return schemaName;
|
37
|
+
}
|
38
|
+
|
39
|
+
@JsonProperty
|
40
|
+
public void setSchemaName(String schemaName)
|
41
|
+
{
|
42
|
+
this.schemaName = schemaName;
|
43
|
+
}
|
44
|
+
|
45
|
+
@JsonProperty
|
46
|
+
public String getTableName() {
|
47
|
+
return tableName;
|
48
|
+
}
|
49
|
+
|
50
|
+
@JsonProperty
|
51
|
+
public void setTableName(String tableName) {
|
52
|
+
this.tableName = tableName;
|
53
|
+
}
|
54
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -19,7 +19,7 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
22
|
+
- classpath/embulk-output-jdbc-0.7.9.jar
|
23
23
|
- lib/embulk/output/jdbc.rb
|
24
24
|
- src/main/java/org/embulk/output/JdbcOutputPlugin.java
|
25
25
|
- src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java
|
@@ -32,6 +32,7 @@ files:
|
|
32
32
|
- src/main/java/org/embulk/output/jdbc/JdbcUtils.java
|
33
33
|
- src/main/java/org/embulk/output/jdbc/MergeConfig.java
|
34
34
|
- src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java
|
35
|
+
- src/main/java/org/embulk/output/jdbc/TableIdentifier.java
|
35
36
|
- src/main/java/org/embulk/output/jdbc/TimestampFormat.java
|
36
37
|
- src/main/java/org/embulk/output/jdbc/ToString.java
|
37
38
|
- src/main/java/org/embulk/output/jdbc/ToStringMap.java
|
Binary file
|