embulk-output-jdbc 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-output-jdbc-0.2.3.jar +0 -0
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +51 -20
- data/src/main/java/org/embulk/output/jdbc/JdbcColumn.java +11 -2
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +14 -3
- data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +7 -1
- data/src/main/java/org/embulk/output/jdbc/setter/ColumnSetter.java +5 -0
- data/src/main/java/org/embulk/output/jdbc/setter/SkipColumnSetter.java +4 -1
- metadata +3 -3
- data/classpath/embulk-output-jdbc-0.2.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ed0f3d732ac1fa2ee1d660480be8587ac137d3f0
|
4
|
+
data.tar.gz: c534795f00578dc8a30e580f9b34a8252e5cc5d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 57157af5fd5be3ceeab92d8dbf36c9f031e2a1ed5bb8f2ae78bf0d91629efdcaf0523de04bbeed799633f40596c4159d8ef589c267b0693d6acd591973b6ee1c
|
7
|
+
data.tar.gz: cbfa1e7646587f442a233f946bde40fe3525414713727333bbe98720408d71067b7e2e60e88358d9d86ac367ed58d1744f4f876e2e7357f5ae3f1009ac5c3786
|
Binary file
|
@@ -9,7 +9,6 @@ import java.util.concurrent.ExecutionException;
|
|
9
9
|
import java.io.IOException;
|
10
10
|
import java.nio.file.Paths;
|
11
11
|
import java.sql.Types;
|
12
|
-
import java.sql.Connection;
|
13
12
|
import java.sql.ResultSet;
|
14
13
|
import java.sql.DatabaseMetaData;
|
15
14
|
import java.sql.SQLException;
|
@@ -32,7 +31,6 @@ import org.embulk.spi.Exec;
|
|
32
31
|
import org.embulk.spi.Column;
|
33
32
|
import org.embulk.spi.ColumnVisitor;
|
34
33
|
import org.embulk.spi.OutputPlugin;
|
35
|
-
import org.embulk.spi.PageOutput;
|
36
34
|
import org.embulk.spi.PluginClassLoader;
|
37
35
|
import org.embulk.spi.Schema;
|
38
36
|
import org.embulk.spi.TransactionalPageOutput;
|
@@ -128,7 +126,12 @@ public abstract class AbstractJdbcOutputPlugin
|
|
128
126
|
|
129
127
|
public boolean isInplace()
|
130
128
|
{
|
131
|
-
return this == INSERT_DIRECT || this == REPLACE_INPLACE;
|
129
|
+
return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
|
130
|
+
}
|
131
|
+
|
132
|
+
public boolean isMerge()
|
133
|
+
{
|
134
|
+
return this == MERGE;
|
132
135
|
}
|
133
136
|
|
134
137
|
public boolean usesMultipleLoadTables()
|
@@ -156,8 +159,11 @@ public abstract class AbstractJdbcOutputPlugin
|
|
156
159
|
case "replace":
|
157
160
|
task.setMode(Mode.REPLACE_INPLACE);
|
158
161
|
break;
|
162
|
+
case "merge":
|
163
|
+
task.setMode(Mode.MERGE);
|
164
|
+
break;
|
159
165
|
default:
|
160
|
-
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace", task.getModeConfig()));
|
166
|
+
throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
|
161
167
|
}
|
162
168
|
|
163
169
|
//switch(task.getModeConfig()) {
|
@@ -383,35 +389,35 @@ public abstract class AbstractJdbcOutputPlugin
|
|
383
389
|
{
|
384
390
|
columns.add(new JdbcColumn(
|
385
391
|
columnName, "BOOLEAN",
|
386
|
-
Types.BOOLEAN, 1, 0));
|
392
|
+
Types.BOOLEAN, 1, 0, false));
|
387
393
|
}
|
388
394
|
|
389
395
|
public void longColumn(Column column)
|
390
396
|
{
|
391
397
|
columns.add(new JdbcColumn(
|
392
398
|
columnName, "BIGINT",
|
393
|
-
Types.BIGINT, 22, 0));
|
399
|
+
Types.BIGINT, 22, 0, false));
|
394
400
|
}
|
395
401
|
|
396
402
|
public void doubleColumn(Column column)
|
397
403
|
{
|
398
404
|
columns.add(new JdbcColumn(
|
399
405
|
columnName, "DOUBLE PRECISION",
|
400
|
-
Types.FLOAT, 24, 0));
|
406
|
+
Types.FLOAT, 24, 0, false));
|
401
407
|
}
|
402
408
|
|
403
409
|
public void stringColumn(Column column)
|
404
410
|
{
|
405
411
|
columns.add(new JdbcColumn(
|
406
412
|
columnName, "CLOB",
|
407
|
-
Types.CLOB, 4000, 0)); // TODO size type param
|
413
|
+
Types.CLOB, 4000, 0, false)); // TODO size type param
|
408
414
|
}
|
409
415
|
|
410
416
|
public void timestampColumn(Column column)
|
411
417
|
{
|
412
418
|
columns.add(new JdbcColumn(
|
413
419
|
columnName, "TIMESTAMP",
|
414
|
-
Types.TIMESTAMP, 26, 0)); // size type param is from postgresql.
|
420
|
+
Types.TIMESTAMP, 26, 0, false)); // size type param is from postgresql.
|
415
421
|
}
|
416
422
|
});
|
417
423
|
}
|
@@ -423,15 +429,27 @@ public abstract class AbstractJdbcOutputPlugin
|
|
423
429
|
{
|
424
430
|
DatabaseMetaData dbm = connection.getMetaData();
|
425
431
|
String escape = dbm.getSearchStringEscape();
|
426
|
-
|
427
|
-
ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
|
428
432
|
String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
|
433
|
+
|
434
|
+
ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
|
435
|
+
ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
|
436
|
+
try {
|
437
|
+
while(rs.next()) {
|
438
|
+
primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
|
439
|
+
}
|
440
|
+
} finally {
|
441
|
+
rs.close();
|
442
|
+
}
|
443
|
+
ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
|
444
|
+
|
429
445
|
String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
|
430
|
-
|
446
|
+
ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
|
447
|
+
rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
|
431
448
|
try {
|
432
449
|
while(rs.next()) {
|
433
450
|
String columnName = rs.getString("COLUMN_NAME");
|
434
451
|
String typeName = rs.getString("TYPE_NAME");
|
452
|
+
boolean isPrimaryKey = primaryKeys.contains(columnName);
|
435
453
|
typeName = typeName.toUpperCase(Locale.ENGLISH);
|
436
454
|
int sqlType = rs.getInt("DATA_TYPE");
|
437
455
|
int colSize = rs.getInt("COLUMN_SIZE");
|
@@ -443,7 +461,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
443
461
|
//rs.getString("COLUMN_DEF") // or null // TODO
|
444
462
|
columns.add(new JdbcColumn(
|
445
463
|
columnName, typeName,
|
446
|
-
sqlType, colSize, decDigit));
|
464
|
+
sqlType, colSize, decDigit, isPrimaryKey));
|
447
465
|
}
|
448
466
|
} finally {
|
449
467
|
rs.close();
|
@@ -522,12 +540,11 @@ public abstract class AbstractJdbcOutputPlugin
|
|
522
540
|
// insert_direct
|
523
541
|
loadTable = task.getTable();
|
524
542
|
}
|
525
|
-
|
526
543
|
b.prepare(loadTable, insertSchema);
|
527
544
|
}
|
528
545
|
});
|
529
546
|
|
530
|
-
PluginPageOutput output =
|
547
|
+
PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(),
|
531
548
|
task.getBatchSize());
|
532
549
|
batch = null;
|
533
550
|
return output;
|
@@ -552,13 +569,20 @@ public abstract class AbstractJdbcOutputPlugin
|
|
552
569
|
return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
|
553
570
|
}
|
554
571
|
|
572
|
+
protected PluginPageOutput newPluginPageOutput(PageReader reader,
|
573
|
+
BatchInsert batch, List<ColumnSetter> columnSetters,
|
574
|
+
int batchSize)
|
575
|
+
{
|
576
|
+
return new PluginPageOutput(reader, batch, columnSetters, batchSize);
|
577
|
+
}
|
578
|
+
|
555
579
|
public static class PluginPageOutput
|
556
580
|
implements TransactionalPageOutput
|
557
581
|
{
|
582
|
+
protected final List<Column> columns;
|
583
|
+
protected final List<ColumnSetter> columnSetters;
|
558
584
|
private final PageReader pageReader;
|
559
585
|
private final BatchInsert batch;
|
560
|
-
private final List<Column> columns;
|
561
|
-
private final List<ColumnSetter> columnSetters;
|
562
586
|
private final int batchSize;
|
563
587
|
private final int foraceBatchFlushSize;
|
564
588
|
|
@@ -583,9 +607,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
583
607
|
if (batch.getBatchWeight() > foraceBatchFlushSize) {
|
584
608
|
batch.flush();
|
585
609
|
}
|
586
|
-
|
587
|
-
columns.get(i).visit(columnSetters.get(i));
|
588
|
-
}
|
610
|
+
handleColumnsSetters();
|
589
611
|
batch.add();
|
590
612
|
}
|
591
613
|
if (batch.getBatchWeight() > batchSize) {
|
@@ -626,6 +648,15 @@ public abstract class AbstractJdbcOutputPlugin
|
|
626
648
|
{
|
627
649
|
return Exec.newCommitReport();
|
628
650
|
}
|
651
|
+
|
652
|
+
protected void handleColumnsSetters()
|
653
|
+
{
|
654
|
+
int size = columnSetters.size();
|
655
|
+
for (int i=0; i < size; i++) {
|
656
|
+
columns.get(i).visit(columnSetters.get(i));
|
657
|
+
}
|
658
|
+
}
|
659
|
+
|
629
660
|
}
|
630
661
|
|
631
662
|
public static interface IdempotentSqlRunnable
|
@@ -11,6 +11,7 @@ public class JdbcColumn
|
|
11
11
|
private int sqlType;
|
12
12
|
private int sizeTypeParameter;
|
13
13
|
private int scaleTypeParameter;
|
14
|
+
private boolean isPrimaryKey;
|
14
15
|
|
15
16
|
@JsonCreator
|
16
17
|
public JdbcColumn(
|
@@ -18,19 +19,21 @@ public class JdbcColumn
|
|
18
19
|
@JsonProperty("typeName") String typeName,
|
19
20
|
@JsonProperty("sqlType") int sqlType,
|
20
21
|
@JsonProperty("sizeTypeParameter") int sizeTypeParameter,
|
21
|
-
@JsonProperty("scaleTypeParameter") int scaleTypeParameter
|
22
|
+
@JsonProperty("scaleTypeParameter") int scaleTypeParameter,
|
23
|
+
@JsonProperty("primaryKey") boolean isPrimaryKey)
|
22
24
|
{
|
23
25
|
this.name = name;
|
24
26
|
this.typeName = typeName;
|
25
27
|
this.sqlType = sqlType;
|
26
28
|
this.sizeTypeParameter = sizeTypeParameter;
|
27
29
|
this.scaleTypeParameter = scaleTypeParameter;
|
30
|
+
this.isPrimaryKey = isPrimaryKey;
|
28
31
|
}
|
29
32
|
|
30
33
|
@JsonIgnore
|
31
34
|
public static JdbcColumn skipColumn()
|
32
35
|
{
|
33
|
-
return new JdbcColumn(null, null, 0, 0, 0);
|
36
|
+
return new JdbcColumn(null, null, 0, 0, 0, false);
|
34
37
|
}
|
35
38
|
|
36
39
|
@JsonIgnore
|
@@ -39,6 +42,12 @@ public class JdbcColumn
|
|
39
42
|
return name == null;
|
40
43
|
}
|
41
44
|
|
45
|
+
@JsonProperty("primaryKey")
|
46
|
+
public boolean isPrimaryKey()
|
47
|
+
{
|
48
|
+
return isPrimaryKey;
|
49
|
+
}
|
50
|
+
|
42
51
|
@JsonProperty("name")
|
43
52
|
public String getName()
|
44
53
|
{
|
@@ -1,6 +1,5 @@
|
|
1
1
|
package org.embulk.output.jdbc;
|
2
2
|
|
3
|
-
import java.util.List;
|
4
3
|
import java.sql.Connection;
|
5
4
|
import java.sql.DatabaseMetaData;
|
6
5
|
import java.sql.PreparedStatement;
|
@@ -66,7 +65,7 @@ public class JdbcOutputConnection
|
|
66
65
|
return rs.next();
|
67
66
|
}
|
68
67
|
}
|
69
|
-
|
68
|
+
|
70
69
|
public void dropTableIfExists(String tableName) throws SQLException
|
71
70
|
{
|
72
71
|
Statement stmt = connection.createStatement();
|
@@ -152,7 +151,7 @@ public class JdbcOutputConnection
|
|
152
151
|
sb.append(buildColumnsOfCreateTableSql(schema));
|
153
152
|
return sb.toString();
|
154
153
|
}
|
155
|
-
|
154
|
+
|
156
155
|
private String buildColumnsOfCreateTableSql(JdbcSchema schema)
|
157
156
|
{
|
158
157
|
StringBuilder sb = new StringBuilder();
|
@@ -306,6 +305,13 @@ public class JdbcOutputConnection
|
|
306
305
|
return connection.prepareStatement(insertSql);
|
307
306
|
}
|
308
307
|
|
308
|
+
public PreparedStatement prepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
309
|
+
{
|
310
|
+
String upsertSql = buildPrepareUpsertSql(toTable, toTableSchema);
|
311
|
+
logger.info("Prepared SQL: {}", upsertSql);
|
312
|
+
return connection.prepareStatement(upsertSql);
|
313
|
+
}
|
314
|
+
|
309
315
|
protected String buildPrepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
310
316
|
{
|
311
317
|
StringBuilder sb = new StringBuilder();
|
@@ -328,6 +334,11 @@ public class JdbcOutputConnection
|
|
328
334
|
return sb.toString();
|
329
335
|
}
|
330
336
|
|
337
|
+
protected String buildPrepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
|
338
|
+
{
|
339
|
+
throw new UnsupportedOperationException("not implemented yet");
|
340
|
+
}
|
341
|
+
|
331
342
|
// TODO
|
332
343
|
//protected void gatherInsertTables(List<String> fromTables, JdbcSchema fromTableSchema, String toTable,
|
333
344
|
// boolean truncateDestinationFirst) throws SQLException
|
@@ -32,13 +32,19 @@ public class StandardBatchInsert
|
|
32
32
|
public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
|
33
33
|
{
|
34
34
|
this.connection = connector.connect(true);
|
35
|
-
this.batch = connection
|
35
|
+
this.batch = newPreparedStatement(connection, loadTable, insertSchema);
|
36
36
|
this.index = 1; // PreparedStatement index begings from 1
|
37
37
|
this.batchRows = 0;
|
38
38
|
this.totalRows = 0;
|
39
39
|
batch.clearBatch();
|
40
40
|
}
|
41
41
|
|
42
|
+
protected PreparedStatement newPreparedStatement(JdbcOutputConnection connection,
|
43
|
+
String loadTable, JdbcSchema insertSchema) throws SQLException
|
44
|
+
{
|
45
|
+
return connection.prepareInsertSql(loadTable, insertSchema);
|
46
|
+
}
|
47
|
+
|
42
48
|
public int getBatchWeight()
|
43
49
|
{
|
44
50
|
return batchWeight;
|
@@ -2,7 +2,6 @@ package org.embulk.output.jdbc.setter;
|
|
2
2
|
|
3
3
|
import org.embulk.spi.PageReader;
|
4
4
|
import org.embulk.spi.time.Timestamp;
|
5
|
-
import org.embulk.output.jdbc.JdbcColumn;
|
6
5
|
import org.embulk.output.jdbc.BatchInsert;
|
7
6
|
|
8
7
|
public class SkipColumnSetter
|
@@ -32,4 +31,8 @@ public class SkipColumnSetter
|
|
32
31
|
protected void timestampValue(Timestamp v)
|
33
32
|
{
|
34
33
|
}
|
34
|
+
|
35
|
+
protected void nullValue()
|
36
|
+
{
|
37
|
+
}
|
35
38
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-04-
|
11
|
+
date: 2015-04-23 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -39,7 +39,7 @@ files:
|
|
39
39
|
- src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java
|
40
40
|
- src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java
|
41
41
|
- src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
|
42
|
-
- classpath/embulk-output-jdbc-0.2.
|
42
|
+
- classpath/embulk-output-jdbc-0.2.3.jar
|
43
43
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
44
44
|
licenses:
|
45
45
|
- Apache 2.0
|
Binary file
|