embulk-output-jdbc 0.2.2 → 0.2.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b00ca29d7e5aac5ac1369fe312807937c0953336
4
- data.tar.gz: 7fd2236f250fd4d24e825a3d974326e60f163b76
3
+ metadata.gz: ed0f3d732ac1fa2ee1d660480be8587ac137d3f0
4
+ data.tar.gz: c534795f00578dc8a30e580f9b34a8252e5cc5d4
5
5
  SHA512:
6
- metadata.gz: 3f09b8a3a96a5df1fb9bb7a00619399fccaca549be880f7729ed155f146f5f6a06ac394cea4187b059b7a938115b9986e18e3ccefef5cccb6a32e1d23c9ea64d
7
- data.tar.gz: 61d57fc29b2816f9809b1dbacc807b55569ec252e4ad55f442b392777738b0732fdf853da1918c7f0f14f8c0e8fd6953676ae35e312ddc3771f47232bbf243b9
6
+ metadata.gz: 57157af5fd5be3ceeab92d8dbf36c9f031e2a1ed5bb8f2ae78bf0d91629efdcaf0523de04bbeed799633f40596c4159d8ef589c267b0693d6acd591973b6ee1c
7
+ data.tar.gz: cbfa1e7646587f442a233f946bde40fe3525414713727333bbe98720408d71067b7e2e60e88358d9d86ac367ed58d1744f4f876e2e7357f5ae3f1009ac5c3786
@@ -9,7 +9,6 @@ import java.util.concurrent.ExecutionException;
9
9
  import java.io.IOException;
10
10
  import java.nio.file.Paths;
11
11
  import java.sql.Types;
12
- import java.sql.Connection;
13
12
  import java.sql.ResultSet;
14
13
  import java.sql.DatabaseMetaData;
15
14
  import java.sql.SQLException;
@@ -32,7 +31,6 @@ import org.embulk.spi.Exec;
32
31
  import org.embulk.spi.Column;
33
32
  import org.embulk.spi.ColumnVisitor;
34
33
  import org.embulk.spi.OutputPlugin;
35
- import org.embulk.spi.PageOutput;
36
34
  import org.embulk.spi.PluginClassLoader;
37
35
  import org.embulk.spi.Schema;
38
36
  import org.embulk.spi.TransactionalPageOutput;
@@ -128,7 +126,12 @@ public abstract class AbstractJdbcOutputPlugin
128
126
 
129
127
  public boolean isInplace()
130
128
  {
131
- return this == INSERT_DIRECT || this == REPLACE_INPLACE;
129
+ return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
130
+ }
131
+
132
+ public boolean isMerge()
133
+ {
134
+ return this == MERGE;
132
135
  }
133
136
 
134
137
  public boolean usesMultipleLoadTables()
@@ -156,8 +159,11 @@ public abstract class AbstractJdbcOutputPlugin
156
159
  case "replace":
157
160
  task.setMode(Mode.REPLACE_INPLACE);
158
161
  break;
162
+ case "merge":
163
+ task.setMode(Mode.MERGE);
164
+ break;
159
165
  default:
160
- throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace", task.getModeConfig()));
166
+ throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
161
167
  }
162
168
 
163
169
  //switch(task.getModeConfig()) {
@@ -383,35 +389,35 @@ public abstract class AbstractJdbcOutputPlugin
383
389
  {
384
390
  columns.add(new JdbcColumn(
385
391
  columnName, "BOOLEAN",
386
- Types.BOOLEAN, 1, 0));
392
+ Types.BOOLEAN, 1, 0, false));
387
393
  }
388
394
 
389
395
  public void longColumn(Column column)
390
396
  {
391
397
  columns.add(new JdbcColumn(
392
398
  columnName, "BIGINT",
393
- Types.BIGINT, 22, 0));
399
+ Types.BIGINT, 22, 0, false));
394
400
  }
395
401
 
396
402
  public void doubleColumn(Column column)
397
403
  {
398
404
  columns.add(new JdbcColumn(
399
405
  columnName, "DOUBLE PRECISION",
400
- Types.FLOAT, 24, 0));
406
+ Types.FLOAT, 24, 0, false));
401
407
  }
402
408
 
403
409
  public void stringColumn(Column column)
404
410
  {
405
411
  columns.add(new JdbcColumn(
406
412
  columnName, "CLOB",
407
- Types.CLOB, 4000, 0)); // TODO size type param
413
+ Types.CLOB, 4000, 0, false)); // TODO size type param
408
414
  }
409
415
 
410
416
  public void timestampColumn(Column column)
411
417
  {
412
418
  columns.add(new JdbcColumn(
413
419
  columnName, "TIMESTAMP",
414
- Types.TIMESTAMP, 26, 0)); // size type param is from postgresql.
420
+ Types.TIMESTAMP, 26, 0, false)); // size type param is from postgresql.
415
421
  }
416
422
  });
417
423
  }
@@ -423,15 +429,27 @@ public abstract class AbstractJdbcOutputPlugin
423
429
  {
424
430
  DatabaseMetaData dbm = connection.getMetaData();
425
431
  String escape = dbm.getSearchStringEscape();
426
-
427
- ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
428
432
  String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
433
+
434
+ ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
435
+ ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
436
+ try {
437
+ while(rs.next()) {
438
+ primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
439
+ }
440
+ } finally {
441
+ rs.close();
442
+ }
443
+ ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
444
+
429
445
  String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
430
- ResultSet rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
446
+ ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
447
+ rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
431
448
  try {
432
449
  while(rs.next()) {
433
450
  String columnName = rs.getString("COLUMN_NAME");
434
451
  String typeName = rs.getString("TYPE_NAME");
452
+ boolean isPrimaryKey = primaryKeys.contains(columnName);
435
453
  typeName = typeName.toUpperCase(Locale.ENGLISH);
436
454
  int sqlType = rs.getInt("DATA_TYPE");
437
455
  int colSize = rs.getInt("COLUMN_SIZE");
@@ -443,7 +461,7 @@ public abstract class AbstractJdbcOutputPlugin
443
461
  //rs.getString("COLUMN_DEF") // or null // TODO
444
462
  columns.add(new JdbcColumn(
445
463
  columnName, typeName,
446
- sqlType, colSize, decDigit));
464
+ sqlType, colSize, decDigit, isPrimaryKey));
447
465
  }
448
466
  } finally {
449
467
  rs.close();
@@ -522,12 +540,11 @@ public abstract class AbstractJdbcOutputPlugin
522
540
  // insert_direct
523
541
  loadTable = task.getTable();
524
542
  }
525
-
526
543
  b.prepare(loadTable, insertSchema);
527
544
  }
528
545
  });
529
546
 
530
- PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters.build(),
547
+ PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(),
531
548
  task.getBatchSize());
532
549
  batch = null;
533
550
  return output;
@@ -552,13 +569,20 @@ public abstract class AbstractJdbcOutputPlugin
552
569
  return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
553
570
  }
554
571
 
572
+ protected PluginPageOutput newPluginPageOutput(PageReader reader,
573
+ BatchInsert batch, List<ColumnSetter> columnSetters,
574
+ int batchSize)
575
+ {
576
+ return new PluginPageOutput(reader, batch, columnSetters, batchSize);
577
+ }
578
+
555
579
  public static class PluginPageOutput
556
580
  implements TransactionalPageOutput
557
581
  {
582
+ protected final List<Column> columns;
583
+ protected final List<ColumnSetter> columnSetters;
558
584
  private final PageReader pageReader;
559
585
  private final BatchInsert batch;
560
- private final List<Column> columns;
561
- private final List<ColumnSetter> columnSetters;
562
586
  private final int batchSize;
563
587
  private final int foraceBatchFlushSize;
564
588
 
@@ -583,9 +607,7 @@ public abstract class AbstractJdbcOutputPlugin
583
607
  if (batch.getBatchWeight() > foraceBatchFlushSize) {
584
608
  batch.flush();
585
609
  }
586
- for (int i=0; i < columnSetters.size(); i++) {
587
- columns.get(i).visit(columnSetters.get(i));
588
- }
610
+ handleColumnsSetters();
589
611
  batch.add();
590
612
  }
591
613
  if (batch.getBatchWeight() > batchSize) {
@@ -626,6 +648,15 @@ public abstract class AbstractJdbcOutputPlugin
626
648
  {
627
649
  return Exec.newCommitReport();
628
650
  }
651
+
652
+ protected void handleColumnsSetters()
653
+ {
654
+ int size = columnSetters.size();
655
+ for (int i=0; i < size; i++) {
656
+ columns.get(i).visit(columnSetters.get(i));
657
+ }
658
+ }
659
+
629
660
  }
630
661
 
631
662
  public static interface IdempotentSqlRunnable
@@ -11,6 +11,7 @@ public class JdbcColumn
11
11
  private int sqlType;
12
12
  private int sizeTypeParameter;
13
13
  private int scaleTypeParameter;
14
+ private boolean isPrimaryKey;
14
15
 
15
16
  @JsonCreator
16
17
  public JdbcColumn(
@@ -18,19 +19,21 @@ public class JdbcColumn
18
19
  @JsonProperty("typeName") String typeName,
19
20
  @JsonProperty("sqlType") int sqlType,
20
21
  @JsonProperty("sizeTypeParameter") int sizeTypeParameter,
21
- @JsonProperty("scaleTypeParameter") int scaleTypeParameter)
22
+ @JsonProperty("scaleTypeParameter") int scaleTypeParameter,
23
+ @JsonProperty("primaryKey") boolean isPrimaryKey)
22
24
  {
23
25
  this.name = name;
24
26
  this.typeName = typeName;
25
27
  this.sqlType = sqlType;
26
28
  this.sizeTypeParameter = sizeTypeParameter;
27
29
  this.scaleTypeParameter = scaleTypeParameter;
30
+ this.isPrimaryKey = isPrimaryKey;
28
31
  }
29
32
 
30
33
  @JsonIgnore
31
34
  public static JdbcColumn skipColumn()
32
35
  {
33
- return new JdbcColumn(null, null, 0, 0, 0);
36
+ return new JdbcColumn(null, null, 0, 0, 0, false);
34
37
  }
35
38
 
36
39
  @JsonIgnore
@@ -39,6 +42,12 @@ public class JdbcColumn
39
42
  return name == null;
40
43
  }
41
44
 
45
+ @JsonProperty("primaryKey")
46
+ public boolean isPrimaryKey()
47
+ {
48
+ return isPrimaryKey;
49
+ }
50
+
42
51
  @JsonProperty("name")
43
52
  public String getName()
44
53
  {
@@ -1,6 +1,5 @@
1
1
  package org.embulk.output.jdbc;
2
2
 
3
- import java.util.List;
4
3
  import java.sql.Connection;
5
4
  import java.sql.DatabaseMetaData;
6
5
  import java.sql.PreparedStatement;
@@ -66,7 +65,7 @@ public class JdbcOutputConnection
66
65
  return rs.next();
67
66
  }
68
67
  }
69
-
68
+
70
69
  public void dropTableIfExists(String tableName) throws SQLException
71
70
  {
72
71
  Statement stmt = connection.createStatement();
@@ -152,7 +151,7 @@ public class JdbcOutputConnection
152
151
  sb.append(buildColumnsOfCreateTableSql(schema));
153
152
  return sb.toString();
154
153
  }
155
-
154
+
156
155
  private String buildColumnsOfCreateTableSql(JdbcSchema schema)
157
156
  {
158
157
  StringBuilder sb = new StringBuilder();
@@ -306,6 +305,13 @@ public class JdbcOutputConnection
306
305
  return connection.prepareStatement(insertSql);
307
306
  }
308
307
 
308
+ public PreparedStatement prepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
309
+ {
310
+ String upsertSql = buildPrepareUpsertSql(toTable, toTableSchema);
311
+ logger.info("Prepared SQL: {}", upsertSql);
312
+ return connection.prepareStatement(upsertSql);
313
+ }
314
+
309
315
  protected String buildPrepareInsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
310
316
  {
311
317
  StringBuilder sb = new StringBuilder();
@@ -328,6 +334,11 @@ public class JdbcOutputConnection
328
334
  return sb.toString();
329
335
  }
330
336
 
337
+ protected String buildPrepareUpsertSql(String toTable, JdbcSchema toTableSchema) throws SQLException
338
+ {
339
+ throw new UnsupportedOperationException("not implemented yet");
340
+ }
341
+
331
342
  // TODO
332
343
  //protected void gatherInsertTables(List<String> fromTables, JdbcSchema fromTableSchema, String toTable,
333
344
  // boolean truncateDestinationFirst) throws SQLException
@@ -32,13 +32,19 @@ public class StandardBatchInsert
32
32
  public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
33
33
  {
34
34
  this.connection = connector.connect(true);
35
- this.batch = connection.prepareInsertSql(loadTable, insertSchema);
35
+ this.batch = newPreparedStatement(connection, loadTable, insertSchema);
36
36
  this.index = 1; // PreparedStatement index begings from 1
37
37
  this.batchRows = 0;
38
38
  this.totalRows = 0;
39
39
  batch.clearBatch();
40
40
  }
41
41
 
42
+ protected PreparedStatement newPreparedStatement(JdbcOutputConnection connection,
43
+ String loadTable, JdbcSchema insertSchema) throws SQLException
44
+ {
45
+ return connection.prepareInsertSql(loadTable, insertSchema);
46
+ }
47
+
42
48
  public int getBatchWeight()
43
49
  {
44
50
  return batchWeight;
@@ -24,6 +24,11 @@ public abstract class ColumnSetter
24
24
  this.column = column;
25
25
  }
26
26
 
27
+ public JdbcColumn getColumn()
28
+ {
29
+ return column;
30
+ }
31
+
27
32
  public int getSqlType()
28
33
  {
29
34
  return column.getSqlType();
@@ -2,7 +2,6 @@ package org.embulk.output.jdbc.setter;
2
2
 
3
3
  import org.embulk.spi.PageReader;
4
4
  import org.embulk.spi.time.Timestamp;
5
- import org.embulk.output.jdbc.JdbcColumn;
6
5
  import org.embulk.output.jdbc.BatchInsert;
7
6
 
8
7
  public class SkipColumnSetter
@@ -32,4 +31,8 @@ public class SkipColumnSetter
32
31
  protected void timestampValue(Timestamp v)
33
32
  {
34
33
  }
34
+
35
+ protected void nullValue()
36
+ {
37
+ }
35
38
  }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.2
4
+ version: 0.2.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-04-07 00:00:00.000000000 Z
11
+ date: 2015-04-23 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -39,7 +39,7 @@ files:
39
39
  - src/main/java/org/embulk/output/jdbc/setter/SqlTimestampColumnSetter.java
40
40
  - src/main/java/org/embulk/output/jdbc/setter/StringColumnSetter.java
41
41
  - src/test/java/org/embulk/output/TestJdbcOutputPlugin.java
42
- - classpath/embulk-output-jdbc-0.2.2.jar
42
+ - classpath/embulk-output-jdbc-0.2.3.jar
43
43
  homepage: https://github.com/embulk/embulk-output-jdbc
44
44
  licenses:
45
45
  - Apache 2.0
Binary file