embulk-output-jdbc 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/embulk-output-jdbc-0.2.3.jar +0 -0
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +51 -20
- data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java +11 -2
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +14 -3
- data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +7 -1
- data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java +5 -0
- data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java +4 -1
- metadata +3 -3
- data/classpath/embulk-output-jdbc-0.2.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed0f3d732ac1fa2ee1d660480be8587ac137d3f0
|
4
|
+
data.tar.gz: c534795f00578dc8a30e580f9b34a8252e5cc5d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 57157af5fd5be3ceeab92d8dbf36c9f031e2a1ed5bb8f2ae78bf0d91629efdcaf0523de04bbeed799633f40596c4159d8ef589c267b0693d6acd591973b6ee1c
|
7
|
+
data.tar.gz: cbfa1e7646587f442a233f946bde40fe3525414713727333bbe98720408d71067b7e2e60e88358d9d86ac367ed58d1744f4f876e2e7357f5ae3f1009ac5c3786
|
Binary file
|
@@ -9,7 +9,6 @@ import java.util.concurrent.ExecutionException;
|
|
9
9
|
import java.io.IOException;
|
10
10
|
import java.nio.file.Paths;
|
11
11
|
import java.sql.Types;
|
12
|
-
import java.sql.Connection;
|
13
12
|
import java.sql.ResultSet;
|
14
13
|
import java.sql.DatabaseMetaData;
|
15
14
|
import java.sql.SQLException;
|
@@ -32,7 +31,6 @@ import org.embulk.spi.Exec;
|
|
32
31
|
import org.embulk.spi.Column;
|
33
32
|
import org.embulk.spi.ColumnVisitor;
|
34
33
|
import org.embulk.spi.OutputPlugin;
|
35
|
-
import org.embulk.spi.PageOutput;
|
36
34
|
import org.embulk.spi.PluginClassLoader;
|
37
35
|
import org.embulk.spi.Schema;
|
38
36
|
import org.embulk.spi.TransactionalPageOutput;
|
@@ -128,7 +126,12 @@ public abstract class AbstractJdbcOutputPlugin
|
|
128
126
|
|
129
127
|
public boolean isInplace()
|
130
128
|
{
|
131
|
-
return this == INSERT_DIRECT || this == REPLACE_INPLACE;
|
129
|
+
return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
|
130
|
+
}
|
131
|
+
|
132
|
+
public boolean isMerge()
|
133
|
+
{
|
134
|
+
return this == MERGE;
|
132
135
|
}
|
133
136
|
|
134
137
|
public boolean usesMultipleLoadTables()
|
@@ -156,8 +159,11 @@ public abstract class AbstractJdbcOutputPlugin
|
|
156
159
|
case "replace":
|
157
160
|
task.setMode(Mode.REPLACE_INPLACE);
|
158
161
|
break;
|
162
|
+
case "merge":
|
163
|
+
task.setMode(Mode.MERGE);
|
164
|
+
break;
|
159
165
|
default:
|
160
|
-
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace", task.getModeConfig()));
|
166
|
+
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
|
161
167
|
}
|
162
168
|
|
163
169
|
//switch(task.getModeConfig()) {
|
@@ -383,35 +389,35 @@ public abstract class AbstractJdbcOutputPlugin
|
|
383
389
|
{
|
384
390
|
columns.add(new JdbcColumn(
|
385
391
|
columnName, "BOOLEAN",
|
386
|
-
Types.BOOLEAN, 1, 0));
|
392
|
+
Types.BOOLEAN, 1, 0, false));
|
387
393
|
}
|
388
394
|
|
389
395
|
public void longColumn(Column column)
|
390
396
|
{
|
391
397
|
columns.add(new JdbcColumn(
|
392
398
|
columnName, "BIGINT",
|
393
|
-
Types.BIGINT, 22, 0));
|
399
|
+
Types.BIGINT, 22, 0, false));
|
394
400
|
}
|
395
401
|
|
396
402
|
public void doubleColumn(Column column)
|
397
403
|
{
|
398
404
|
columns.add(new JdbcColumn(
|
399
405
|
columnName, "DOUBLE PRECISION",
|
400
|
-
Types.FLOAT, 24, 0));
|
406
|
+
Types.FLOAT, 24, 0, false));
|
401
407
|
}
|
402
408
|
|
403
409
|
public void stringColumn(Column column)
|
404
410
|
{
|
405
411
|
columns.add(new JdbcColumn(
|
406
412
|
columnName, "CLOB",
|
407
|
-
Types.CLOB, 4000, 0)); // TODO size type param
|
413
|
+
Types.CLOB, 4000, 0, false)); // TODO size type param
|
408
414
|
}
|
409
415
|
|
410
416
|
public void timestampColumn(Column column)
|
411
417
|
{
|
412
418
|
columns.add(new JdbcColumn(
|
413
419
|
columnName, "TIMESTAMP",
|
414
|
-
Types.TIMESTAMP, 26, 0)); // size type param is from postgresql.
|
420
|
+
Types.TIMESTAMP, 26, 0, false)); // size type param is from postgresql.
|
415
421
|
}
|
416
422
|
});
|
417
423
|
}
|
@@ -423,15 +429,27 @@ public abstract class AbstractJdbcOutputPlugin
|
|
423
429
|
{
|
424
430
|
DatabaseMetaData dbm = connection.getMetaData();
|
425
431
|
String escape = dbm.getSearchStringEscape();
|
426
|
-
|
427
|
-
ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
|
428
432
|
String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
|
433
|
+
|
434
|
+
ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
|
435
|
+
ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
|
436
|
+
try {
|
437
|
+
while(rs.next()) {
|
438
|
+
primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
|
439
|
+
}
|
440
|
+
} finally {
|
441
|
+
rs.close();
|
442
|
+
}
|
443
|
+
ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
|
444
|
+
|
429
445
|
String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
|
430
|
-
|
446
|
+
ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
|
447
|
+
rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
|
431
448
|
try {
|
432
449
|
while(rs.next()) {
|
433
450
|
String columnName = rs.getString("COLUMN_NAME");
|
434
451
|
String typeName = rs.getString("TYPE_NAME");
|
452
|
+
boolean isPrimaryKey = primaryKeys.contains(columnName);
|
435
453
|
typeName = typeName.toUpperCase(Locale.ENGLISH);
|
436
454
|
int sqlType = rs.getInt("DATA_TYPE");
|
437
455
|
int colSize = rs.getInt("COLUMN_SIZE");
|
@@ -443,7 +461,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
443
461
|
//rs.getString("COLUMN_DEF") // or null // TODO
|
444
462
|
columns.add(new JdbcColumn(
|
445
463
|
columnName, typeName,
|
446
|
-
sqlType, colSize, decDigit));
|
464
|
+
sqlType, colSize, decDigit, isPrimaryKey));
|
447
465
|
}
|
448
466
|
} finally {
|
449
467
|
rs.close();
|
@@ -522,12 +540,11 @@ public abstract class AbstractJdbcOutputPlugin
|
|
522
540
|
// insert_direct
|
523
541
|
loadTable = task.getTable();
|
524
542
|
}
|
525
|
-
|
526
543
|
b.prepare(loadTable, insertSchema);
|
527
544
|
}
|
528
545
|
});
|
529
546
|
|
530
|
-
PluginPageOutput output =
|
547
|
+
PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(),
|
531
548
|
task.getBatchSize());
|
532
549
|
batch = null;
|
533
550
|
return output;
|
@@ -552,13 +569,20 @@ public abstract class AbstractJdbcOutputPlugin
|
|
552
569
|
return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
|
553
570
|
}
|
554
571
|
|
572
|
+
protected PluginPageOutput newPluginPageOutput(PageReader reader,
|
573
|
+
BatchInsert batch, List<ColumnSetter> columnSetters,
|
574
|
+
int batchSize)
|
575
|
+
{
|
576
|
+
return new PluginPageOutput(reader, batch, columnSetters, batchSize);
|
577
|
+
}
|
578
|
+
|
555
579
|
public static class PluginPageOutput
|
556
580
|
implements TransactionalPageOutput
|
557
581
|
{
|
582
|
+
protected final List<Column> columns;
|
583
|
+
protected final List<ColumnSetter> columnSetters;
|
558
584
|
private final PageReader pageReader;
|
559
585
|
private final BatchInsert batch;
|
560
|
-
private final List<Column> columns;
|
561
|
-
private final List<ColumnSetter> columnSetters;
|
562
586
|
private final int batchSize;
|
563
587
|
private final int foraceBatchFlushSize;
|
564
588
|
|
@@ -583,9 +607,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
583
607
|
if (batch.getBatchWeight() > foraceBatchFlushSize) {
|
584
608
|
batch.flush();
|
585
609
|
}
|
586
|
-
|
587
|
-
columns.get(i).visit(columnSetters.get(i));
|
588
|
-
}
|
610
|
+
handleColumnsSetters();
|
589
611
|
batch.add();
|
590
612
|
}
|
591
613
|
if (batch.getBatchWeight() > batchSize) {
|
@@ -626,6 +648,15 @@ public abstract class AbstractJdbcOutputPlugin
|
|
626
648
|
{
|
627
649
|
return Exec.newCommitReport();
|
628
650
|
}
|
651
|
+
|
652
|
+
protected void handleColumnsSetters()
|
653
|
+
{
|
654
|
+
int size = columnSetters.size();
|
655
|
+
for (int i=0; i < size; i++) {
|
656
|
+
columns.get(i).visit(columnSetters.get(i));
|
657
|
+
}
|
658
|
+
}
|
659
|
+
|
629
660
|
}
|
630
661
|
|
631
662
|
public static interface IdempotentSqlRunnable
|
@@ -11,6 +11,7 @@ public class JdbcColumn
|
|
11
11
|
private int sqlType;
|
12
12
|
private int sizeTypeParameter;
|
13
13
|
private int scaleTypeParameter;
|
14
|
+
private boolean isPrimaryKey;
|
14
15
|
|
15
16
|
@JsonCreator
|
16
17
|
public JdbcColumn(
|
@@ -18,19 +19,21 @@ public class JdbcColumn
|
|
18
19
|
@JsonProperty("typeName") String typeName,
|
19
20
|
@JsonProperty("sqlType") int sqlType,
|
20
21
|
@JsonProperty("sizeTypeParameter") int sizeTypeParameter,
|
21
|
-
@JsonProperty("scaleTypeParameter") int scaleTypeParameter
|
22
|
+
@JsonProperty("scaleTypeParameter") int scaleTypeParameter,
|
23
|
+
@JsonProperty("primaryKey") boolean isPrimaryKey)
|
22
24
|
{
|
23
25
|
this.name = name;
|
24
26
|
this.typeName = typeName;
|
25
27
|
this.sqlType = sqlType;
|
26
28
|
this.sizeTypeParameter = sizeTypeParameter;
|
27
29
|
this.scaleTypeParameter = scaleTypeParameter;
|
30
|
+
this.isPrimaryKey = isPrimaryKey;
|
28
31
|
}
|
29
32
|
|
30
33
|
@JsonIgnore
|
31
34
|
public static JdbcColumn skipColumn()
|
32
35
|
{
|
33
|
-
return new JdbcColumn(null, null, 0, 0, 0);
|
36
|
+
return new JdbcColumn(null, null, 0, 0, 0, false);
|
34
37
|
}
|
35
38
|
|
36
39
|
@JsonIgnore
|
@@ -39,6 +42,12 @@ public class JdbcColumn
|
|
39
42
|
return name == null;
|
40
43
|
}
|
41
44
|
|
45
|
+
@JsonProperty("primaryKey")
|
46
|
+
public boolean isPrimaryKey()
|
47
|
+
{
|
48
|
+
return isPrimaryKey;
|
49
|
+
}
|
50
|
+
|
42
51
|
@JsonProperty("name")
|
43
52
|
public String getName()
|
44
53
|
{
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.output.jdbc;
|
2
2
|
|
3
|
-
import java.util.List;
|
4
3
|
import java.sql.Connection;
|
5
4
|
import java.sql.DatabaseMetaData;
|
6
5
|
import java.sql.PreparedStatement;
|
@@ -66,7 +65,7 @@ public class JdbcOutputConnection
|
|
66
65
|
return rs.next();
|
67
66
|
}
|
68
67
|
}
|
69
|
-
|
68
|
+
|
70
69
|
public void dropTableIfExists(String tableName) throws SQLException
|
71
70
|
{
|
72
71
|
Statement stmt = connection.createStatement();
|
@@ -152,7 +151,7 @@ public class JdbcOutputConnection
|
|
152
151
|
sb.append(buildColumnsOfCreateTableSql(schema));
|
153
152
|
return sb.toString();
|
154
153
|
}
|
155
|
-
|
154
|
+
|
156
155
|
private String buildColumnsOfCreateTableSql(JdbcSchema schema)
|
157
156
|
{
|
158
157
|
StringBuilder sb = new StringBuilder();
|
@@ -306,6 +305,13 @@ public class JdbcOutputConnection
|
|
306
305
|
return connection.prepareStatement(insertSql);
|
307
306
|
}
|
308
307
|
|
308
|
+
public PreparedStatement prepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
309
|
+
{
|
310
|
+
String upsertSql = buildPrepareUpsertSql(toTable, toTableSchema);
|
311
|
+
logger.info("Prepared SQL: {}", upsertSql);
|
312
|
+
return connection.prepareStatement(upsertSql);
|
313
|
+
}
|
314
|
+
|
309
315
|
protected String buildPrepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
310
316
|
{
|
311
317
|
StringBuilder sb = new StringBuilder();
|
@@ -328,6 +334,11 @@ public class JdbcOutputConnection
|
|
328
334
|
return sb.toString();
|
329
335
|
}
|
330
336
|
|
337
|
+
protected String buildPrepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
338
|
+
{
|
339
|
+
throw new UnsupportedOperationException("not implemented yet");
|
340
|
+
}
|
341
|
+
|
331
342
|
// TODO
|
332
343
|
//protected void gatherInsertTables(List<String> fromTables, JdbcSchema fromTableSchema, String toTable,
|
333
344
|
// boolean truncateDestinationFirst) throws SQLException
|
@@ -32,13 +32,19 @@ public class StandardBatchInsert
|
|
32
32
|
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
33
33
|
{
|
34
34
|
this.connection = connector.connect(true);
|
35
|
-
this.batch = connection
|
35
|
+
this.batch = newPreparedStatement(connection, loadTable, insertSchema);
|
36
36
|
this.index = 1; // PreparedStatement index begings from 1
|
37
37
|
this.batchRows = 0;
|
38
38
|
this.totalRows = 0;
|
39
39
|
batch.clearBatch();
|
40
40
|
}
|
41
41
|
|
42
|
+
protected PreparedStatement newPreparedStatement(JdbcOutputConnection connection,
|
43
|
+
String loadTable, JdbcSchema insertSchema) throws SQLException
|
44
|
+
{
|
45
|
+
return connection.prepareInsertSql(loadTable, insertSchema);
|
46
|
+
}
|
47
|
+
|
42
48
|
public int getBatchWeight()
|
43
49
|
{
|
44
50
|
return batchWeight;
|
@@ -2,7 +2,6 @@ package org.embulk.output.jdbc.setter;
|
|
2
2
|
|
3
3
|
import org.embulk.spi.PageReader;
|
4
4
|
import org.embulk.spi.time.Timestamp;
|
5
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
6
5
|
import org.embulk.output.jdbc.BatchInsert;
|
7
6
|
|
8
7
|
public class SkipColumnSetter
|
@@ -32,4 +31,8 @@ public class SkipColumnSetter
|
|
32
31
|
protected void timestampValue(Timestamp v)
|
33
32
|
{
|
34
33
|
}
|
34
|
+
|
35
|
+
protected void nullValue()
|
36
|
+
{
|
37
|
+
}
|
35
38
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -39,7 +39,7 @@ files:
|
|
39
39
|
- src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java
|
40
40
|
- src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java
|
41
41
|
- src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
|
42
|
-
- classpath/embulk-output-jdbc-0.2.
|
42
|
+
- classpath/embulk-output-jdbc-0.2.3.jar
|
43
43
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
44
44
|
licenses:
|
45
45
|
- Apache 2.0
|
Binary file
|