embulk-output-jdbc 0.6.0 → 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-output-jdbc-0.6.1.jar +0 -0
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +48 -22
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +3 -1
- data/src/main/java/org/embulk/output/jdbc/JdbcSchema.java +19 -0
- data/src/main/java/org/embulk/output/jdbc/StandardBatchInsert.java +1 -1
- metadata +3 -3
- data/classpath/embulk-output-jdbc-0.6.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 55c72f27b523c6e9e8a9f4bdcc6e34238fc2ab93
|
4
|
+
data.tar.gz: 1d6350d92b143a0423c2b5de300853b2f92ca9e1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d625cc29b72e815c187745ee8553761bf3e27408c98c3ce69cb7539810ad5b32dc390e3e6557cd65954c0102cbb02569b1031f85d4c6ebd50d85d27abb32ed86
|
7
|
+
data.tar.gz: f8adc582282ef5f15e71401da3650162cf965dd7b7ae38145b7139dfa8dcbfd49e241eb45acec9a43a122ce45371127934bed643f4edc891bf8af4c5bdcf9da0
|
Binary file
|
@@ -91,6 +91,9 @@ public abstract class AbstractJdbcOutputPlugin
|
|
91
91
|
@ConfigDefault("\"UTC\"")
|
92
92
|
public DateTimeZone getDefaultTimeZone();
|
93
93
|
|
94
|
+
public void setActualTable(String actualTable);
|
95
|
+
public String getActualTable();
|
96
|
+
|
94
97
|
public void setMergeKeys(Optional<List<String>> keys);
|
95
98
|
|
96
99
|
public void setFeatures(Features features);
|
@@ -423,16 +426,37 @@ public abstract class AbstractJdbcOutputPlugin
|
|
423
426
|
PluginTask task, final Schema schema, int taskCount) throws SQLException
|
424
427
|
{
|
425
428
|
if (schema.getColumnCount() == 0) {
|
426
|
-
throw new ConfigException("
|
429
|
+
throw new ConfigException("No column.");
|
427
430
|
}
|
428
431
|
|
429
432
|
Mode mode = task.getMode();
|
430
433
|
logger.info("Using {} mode", mode);
|
431
434
|
|
435
|
+
if (con.tableExists(task.getTable())) {
|
436
|
+
task.setActualTable(task.getTable());
|
437
|
+
} else {
|
438
|
+
String upperTable = task.getTable().toUpperCase();
|
439
|
+
String lowerTable = task.getTable().toLowerCase();
|
440
|
+
if (con.tableExists(upperTable)) {
|
441
|
+
if (con.tableExists(lowerTable)) {
|
442
|
+
throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
|
443
|
+
task.getTable(), upperTable, lowerTable));
|
444
|
+
} else {
|
445
|
+
task.setActualTable(upperTable);
|
446
|
+
}
|
447
|
+
} else {
|
448
|
+
if (con.tableExists(lowerTable)) {
|
449
|
+
task.setActualTable(lowerTable);
|
450
|
+
} else {
|
451
|
+
task.setActualTable(task.getTable());
|
452
|
+
}
|
453
|
+
}
|
454
|
+
}
|
455
|
+
|
432
456
|
Optional<JdbcSchema> initialTargetTableSchema =
|
433
457
|
mode.ignoreTargetTableSchema() ?
|
434
458
|
Optional.<JdbcSchema>absent() :
|
435
|
-
newJdbcSchemaFromTableIfExists(con, task.
|
459
|
+
newJdbcSchemaFromTableIfExists(con, task.getActualTable());
|
436
460
|
|
437
461
|
// TODO get CREATE TABLE statement from task if set
|
438
462
|
JdbcSchema newTableSchema = applyColumnOptionsToNewTableSchema(
|
@@ -449,13 +473,13 @@ public abstract class AbstractJdbcOutputPlugin
|
|
449
473
|
// direct modify mode doesn't need intermediate tables.
|
450
474
|
ImmutableList.Builder<String> intermTableNames = ImmutableList.builder();
|
451
475
|
if (mode.tempTablePerTask()) {
|
452
|
-
String namePrefix = generateIntermediateTableNamePrefix(task.
|
476
|
+
String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable(), con, 3,
|
453
477
|
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
454
478
|
for (int i=0; i < taskCount; i++) {
|
455
479
|
intermTableNames.add(namePrefix + String.format("%03d", i));
|
456
480
|
}
|
457
481
|
} else {
|
458
|
-
String name = generateIntermediateTableNamePrefix(task.
|
482
|
+
String name = generateIntermediateTableNamePrefix(task.getActualTable(), con, 0,
|
459
483
|
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
460
484
|
intermTableNames.add(name);
|
461
485
|
}
|
@@ -483,8 +507,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
483
507
|
} else {
|
484
508
|
// also create the target table if not exists
|
485
509
|
// CREATE TABLE IF NOT EXISTS xyz
|
486
|
-
con.createTableIfNotExists(task.
|
487
|
-
targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.
|
510
|
+
con.createTableIfNotExists(task.getActualTable(), newTableSchema);
|
511
|
+
targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getActualTable()).get();
|
488
512
|
task.setNewTableSchema(Optional.<JdbcSchema>absent());
|
489
513
|
}
|
490
514
|
task.setTargetTableSchema(matchSchemaByColumnNames(schema, targetTableSchema));
|
@@ -574,7 +598,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
574
598
|
return new JdbcSchema(Lists.transform(schema.getColumns(), new Function<JdbcColumn, JdbcColumn>() {
|
575
599
|
public JdbcColumn apply(JdbcColumn c)
|
576
600
|
{
|
577
|
-
JdbcColumnOption option = columnOptionOf(columnOptions, c);
|
601
|
+
JdbcColumnOption option = columnOptionOf(columnOptions, c.getName());
|
578
602
|
if (option.getType().isPresent()) {
|
579
603
|
return JdbcColumn.newTypeDeclaredColumn(
|
580
604
|
c.getName(), Types.OTHER, // sqlType, isNotNull, and isUniqueKey are ignored
|
@@ -590,23 +614,22 @@ public abstract class AbstractJdbcOutputPlugin
|
|
590
614
|
Map<String, JdbcColumnOption> columnOptions)
|
591
615
|
{
|
592
616
|
ImmutableList.Builder<ColumnSetter> builder = ImmutableList.builder();
|
593
|
-
int schemaColumnIndex = 0;
|
594
|
-
|
617
|
+
for (int schemaColumnIndex = 0; schemaColumnIndex < targetTableSchema.getCount(); schemaColumnIndex++) {
|
618
|
+
JdbcColumn targetColumn = targetTableSchema.getColumn(schemaColumnIndex);
|
619
|
+
Column inputColumn = inputValueSchema.getColumn(schemaColumnIndex);
|
595
620
|
if (targetColumn.isSkipColumn()) {
|
596
621
|
builder.add(factory.newSkipColumnSetter());
|
597
622
|
} else {
|
598
|
-
|
599
|
-
JdbcColumnOption option = columnOptionOf(columnOptions, targetColumn);
|
623
|
+
JdbcColumnOption option = columnOptionOf(columnOptions, inputColumn.getName());
|
600
624
|
builder.add(factory.newColumnSetter(targetColumn, option));
|
601
|
-
schemaColumnIndex++;
|
602
625
|
}
|
603
626
|
}
|
604
627
|
return builder.build();
|
605
628
|
}
|
606
629
|
|
607
|
-
private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions,
|
630
|
+
private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, String columnName)
|
608
631
|
{
|
609
|
-
return Optional.fromNullable(columnOptions.get(
|
632
|
+
return Optional.fromNullable(columnOptions.get(columnName)).or(
|
610
633
|
// default column option
|
611
634
|
new Supplier<JdbcColumnOption>()
|
612
635
|
{
|
@@ -641,30 +664,30 @@ public abstract class AbstractJdbcOutputPlugin
|
|
641
664
|
case INSERT:
|
642
665
|
// aggregate insert into target
|
643
666
|
if (task.getNewTableSchema().isPresent()) {
|
644
|
-
con.createTableIfNotExists(task.
|
667
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
|
645
668
|
}
|
646
|
-
con.collectInsert(task.getIntermediateTables().get(), schema, task.
|
669
|
+
con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), false);
|
647
670
|
break;
|
648
671
|
|
649
672
|
case TRUNCATE_INSERT:
|
650
673
|
// truncate & aggregate insert into target
|
651
674
|
if (task.getNewTableSchema().isPresent()) {
|
652
|
-
con.createTableIfNotExists(task.
|
675
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
|
653
676
|
}
|
654
|
-
con.collectInsert(task.getIntermediateTables().get(), schema, task.
|
677
|
+
con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), true);
|
655
678
|
break;
|
656
679
|
|
657
680
|
case MERGE:
|
658
681
|
// aggregate merge into target
|
659
682
|
if (task.getNewTableSchema().isPresent()) {
|
660
|
-
con.createTableIfNotExists(task.
|
683
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
|
661
684
|
}
|
662
|
-
con.collectMerge(task.getIntermediateTables().get(), schema, task.
|
685
|
+
con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(), task.getMergeKeys().get());
|
663
686
|
break;
|
664
687
|
|
665
688
|
case REPLACE:
|
666
689
|
// swap table
|
667
|
-
con.replaceTable(task.getIntermediateTables().get().get(0), schema, task.
|
690
|
+
con.replaceTable(task.getIntermediateTables().get().get(0), schema, task.getActualTable());
|
668
691
|
break;
|
669
692
|
}
|
670
693
|
}
|
@@ -823,13 +846,16 @@ public abstract class AbstractJdbcOutputPlugin
|
|
823
846
|
task.getTargetTableSchema(), schema,
|
824
847
|
task.getColumnOptions());
|
825
848
|
JdbcSchema insertIntoSchema = filterSkipColumns(task.getTargetTableSchema());
|
849
|
+
if (insertIntoSchema.getCount() == 0) {
|
850
|
+
throw new SQLException("No column to insert.");
|
851
|
+
}
|
826
852
|
|
827
853
|
// configure BatchInsert -> an intermediate table (!isDirectModify) or the target table (isDirectModify)
|
828
854
|
String destTable;
|
829
855
|
if (mode.tempTablePerTask()) {
|
830
856
|
destTable = task.getIntermediateTables().get().get(taskIndex);
|
831
857
|
} else if (mode.isDirectModify()) {
|
832
|
-
destTable = task.
|
858
|
+
destTable = task.getActualTable();
|
833
859
|
} else {
|
834
860
|
destTable = task.getIntermediateTables().get().get(0);
|
835
861
|
}
|
@@ -1,6 +1,9 @@
|
|
1
1
|
package org.embulk.output.jdbc;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
+
|
5
|
+
import org.embulk.config.ConfigException;
|
6
|
+
|
4
7
|
import com.google.common.base.Optional;
|
5
8
|
import com.google.common.collect.ImmutableList;
|
6
9
|
import com.fasterxml.jackson.annotation.JsonCreator;
|
@@ -24,11 +27,27 @@ public class JdbcSchema
|
|
24
27
|
|
25
28
|
public Optional<JdbcColumn> findColumn(String name)
|
26
29
|
{
|
30
|
+
// because both upper case column and lower case column may exist, search twice
|
27
31
|
for (JdbcColumn column : columns) {
|
28
32
|
if (column.getName().equals(name)) {
|
29
33
|
return Optional.of(column);
|
30
34
|
}
|
31
35
|
}
|
36
|
+
|
37
|
+
JdbcColumn foundColumn = null;
|
38
|
+
for (JdbcColumn column : columns) {
|
39
|
+
if (column.getName().equalsIgnoreCase(name)) {
|
40
|
+
if (foundColumn != null) {
|
41
|
+
throw new ConfigException(String.format("Cannot specify column '%s' because both '%s' and '%s' exist.",
|
42
|
+
name, foundColumn.getName(), column.getName()));
|
43
|
+
}
|
44
|
+
foundColumn = column;
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
if (foundColumn != null) {
|
49
|
+
return Optional.of(foundColumn);
|
50
|
+
}
|
32
51
|
return Optional.absent();
|
33
52
|
}
|
34
53
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-06-23 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -58,7 +58,7 @@ files:
|
|
58
58
|
- src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
|
59
59
|
- src/test/java/org/embulk/output/TimestampFormatTest.java
|
60
60
|
- src/test/java/org/embulk/output/tester/EmbulkPluginTester.java
|
61
|
-
- classpath/embulk-output-jdbc-0.6.
|
61
|
+
- classpath/embulk-output-jdbc-0.6.1.jar
|
62
62
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
63
63
|
licenses:
|
64
64
|
- Apache 2.0
|
Binary file
|