embulk-output-jdbc 0.7.12 → 0.7.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ae6b599749ebe50a9b832c6ba8d0784a6737cb7c
4
- data.tar.gz: 43c7475f21f28132563ba0a6592e7fbedb4790e2
3
+ metadata.gz: 411a4c67839886d900bdf7bb5a47216f365103f4
4
+ data.tar.gz: 53967bc25669370087fa99085b5975785a093630
5
5
  SHA512:
6
- metadata.gz: 5bb01073bf551b713858e89f0f7364c428457c51107c25f357d962573ca5556f3623d0d4462b0d44fb75506c936853996c81684c639c7073b0e9921cb54be33b
7
- data.tar.gz: 468e740668ce4e9f03edfc001703fcb21017f5380f9fe00f3de71c1de53fe35ce4ee580517c5f8a07cd45ee6fd165633b6994a4999bf2a5db039a09101094ae3
6
+ metadata.gz: 461685691438c570b0ffebd33954cfb24f086b39cac220ecc7483aa0ed678af5f75240019f5f184f0dd5119180c1147e2eba43e3a6e1fcdb39b1331644502093
7
+ data.tar.gz: 4ed522ff2ae37f35e95c91122e3a146a24e095f9c74f821ded1ee6110c1affbeafc68cfb79c485c37b6750ba7118d15a2d6f63fd69b68db1b0c302819ba3a893
data/README.md CHANGED
@@ -17,6 +17,8 @@ Generic JDBC output plugin for Embulk loads records to a database using a JDBC d
17
17
  - **password**: database login password (string, optional)
18
18
  - **schema**: destination schema name (string, default: use default schema)
19
19
  - **table**: destination table name (string, required)
20
+ - **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
21
+ - **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
20
22
  - **options**: extra JDBC properties (hash, default: {})
21
23
  - **retry_limit** max retry count for database operations (integer, default: 12)
22
24
  - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
@@ -67,7 +67,7 @@ public class JdbcOutputPlugin
67
67
  GenericPluginTask t = (GenericPluginTask) task;
68
68
 
69
69
  if (t.getDriverPath().isPresent()) {
70
- loadDriverJar(t.getDriverPath().get());
70
+ addDriverJarToClasspath(t.getDriverPath().get());
71
71
  }
72
72
 
73
73
  Properties props = new Properties();
@@ -1,13 +1,18 @@
1
1
  package org.embulk.output.jdbc;
2
2
 
3
- import java.util.HashSet;
4
3
  import java.util.List;
5
4
  import java.util.Map;
6
5
  import java.util.Locale;
7
6
  import java.util.Set;
8
7
  import java.util.concurrent.ExecutionException;
8
+ import java.io.File;
9
+ import java.io.FileFilter;
9
10
  import java.io.IOException;
11
+ import java.net.MalformedURLException;
12
+ import java.net.URISyntaxException;
13
+ import java.net.URL;
10
14
  import java.nio.charset.Charset;
15
+ import java.nio.file.Path;
11
16
  import java.nio.file.Paths;
12
17
  import java.sql.Types;
13
18
  import java.sql.ResultSet;
@@ -47,7 +52,6 @@ import org.embulk.spi.Schema;
47
52
  import org.embulk.spi.TransactionalPageOutput;
48
53
  import org.embulk.spi.Page;
49
54
  import org.embulk.spi.PageReader;
50
- import org.embulk.spi.time.Timestamp;
51
55
  import org.embulk.output.jdbc.setter.ColumnSetter;
52
56
  import org.embulk.output.jdbc.setter.ColumnSetterFactory;
53
57
  import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
@@ -59,8 +63,6 @@ import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
59
63
  public abstract class AbstractJdbcOutputPlugin
60
64
  implements OutputPlugin
61
65
  {
62
- private final static Set<String> loadedJarGlobs = new HashSet<String>();
63
-
64
66
  protected final Logger logger = Exec.getLogger(getClass());
65
67
 
66
68
  public interface PluginTask
@@ -89,6 +91,14 @@ public abstract class AbstractJdbcOutputPlugin
89
91
  @ConfigDefault("{}")
90
92
  public Map<String, JdbcColumnOption> getColumnOptions();
91
93
 
94
+ @Config("create_table_constraint")
95
+ @ConfigDefault("null")
96
+ public Optional<String> getCreateTableConstraint();
97
+
98
+ @Config("create_table_option")
99
+ @ConfigDefault("null")
100
+ public Optional<String> getCreateTableOption();
101
+
92
102
  @Config("default_timezone")
93
103
  @ConfigDefault("\"UTC\"")
94
104
  public DateTimeZone getDefaultTimeZone();
@@ -217,16 +227,52 @@ public abstract class AbstractJdbcOutputPlugin
217
227
  }
218
228
  }
219
229
 
220
- protected void loadDriverJar(String glob)
230
+ protected void addDriverJarToClasspath(String glob)
221
231
  {
222
- synchronized (loadedJarGlobs) {
223
- if (!loadedJarGlobs.contains(glob)) {
224
- // TODO match glob
225
- PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
226
- loader.addPath(Paths.get(glob));
227
- loadedJarGlobs.add(glob);
232
+ // TODO match glob
233
+ PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
234
+ Path path = Paths.get(glob);
235
+ if (!path.toFile().exists()) {
236
+ throw new ConfigException("The specified driver jar doesn't exist: " + glob);
237
+ }
238
+ loader.addPath(Paths.get(glob));
239
+ }
240
+
241
+ protected void loadDriver(String className, Optional<String> driverPath)
242
+ {
243
+ if (driverPath.isPresent()) {
244
+ addDriverJarToClasspath(driverPath.get());
245
+ } else {
246
+ try {
247
+ // Gradle test task will add JDBC driver to classpath
248
+ Class.forName(className);
249
+
250
+ } catch (ClassNotFoundException ex) {
251
+ File root = findPluginRoot(getClass());
252
+ File driverLib = new File(root, "default_jdbc_driver");
253
+ File[] files = driverLib.listFiles(new FileFilter() {
254
+ @Override
255
+ public boolean accept(File file) {
256
+ return file.isFile() && file.getName().endsWith(".jar");
257
+ }
258
+ });
259
+ if (files == null || files.length == 0) {
260
+ throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
261
+ } else {
262
+ for (File file : files) {
263
+ logger.info("JDBC Driver = " + file.getAbsolutePath());
264
+ addDriverJarToClasspath(file.getAbsolutePath());
265
+ }
266
+ }
228
267
  }
229
268
  }
269
+
270
+ // Load JDBC Driver
271
+ try {
272
+ Class.forName(className);
273
+ } catch (ClassNotFoundException ex) {
274
+ throw new RuntimeException(ex);
275
+ }
230
276
  }
231
277
 
232
278
  // for subclasses to add @Config
@@ -372,13 +418,6 @@ public abstract class AbstractJdbcOutputPlugin
372
418
  return commit(task, schema, taskCount);
373
419
  }
374
420
 
375
- protected String getTransactionUniqueName()
376
- {
377
- // TODO use uuid?
378
- Timestamp t = Exec.session().getTransactionTime();
379
- return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
380
- }
381
-
382
421
  private PluginTask begin(final PluginTask task,
383
422
  final Schema schema, final int taskCount)
384
423
  {
@@ -525,7 +564,7 @@ public abstract class AbstractJdbcOutputPlugin
525
564
  } else {
526
565
  // also create the target table if not exists
527
566
  // CREATE TABLE IF NOT EXISTS xyz
528
- con.createTableIfNotExists(task.getActualTable(), newTableSchema);
567
+ con.createTableIfNotExists(task.getActualTable(), newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
529
568
  targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getActualTable()).get();
530
569
  task.setNewTableSchema(Optional.<JdbcSchema>absent());
531
570
  }
@@ -598,17 +637,17 @@ public abstract class AbstractJdbcOutputPlugin
598
637
  String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 3,
599
638
  task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
600
639
  for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
601
- String tableName = namePrefix + String.format("%03d", taskIndex);
640
+ String tableName = namePrefix + String.format("%03d", taskIndex % 1000);
602
641
  table = buildIntermediateTableId(con, task, tableName);
603
642
  // if table already exists, SQLException will be thrown
604
- con.createTable(table, newTableSchema);
643
+ con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
605
644
  intermTables.add(table);
606
645
  }
607
646
  } else {
608
647
  String tableName = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 0,
609
648
  task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
610
649
  table = buildIntermediateTableId(con, task, tableName);
611
- con.createTable(table, newTableSchema);
650
+ con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
612
651
  intermTables.add(table);
613
652
  }
614
653
  return intermTables.build();
@@ -667,8 +706,8 @@ public abstract class AbstractJdbcOutputPlugin
667
706
  {
668
707
  Charset tableNameCharset = con.getTableNameCharset();
669
708
  String tableName = baseTableName;
670
- String suffix = "_bl_tmp";
671
- String uniqueSuffix = getTransactionUniqueName() + suffix;
709
+ String suffix = "_embulk";
710
+ String uniqueSuffix = String.format("%016x", System.currentTimeMillis()) + suffix;
672
711
 
673
712
  // way to count length of table name varies by DBMSs (bytes or characters),
674
713
  // so truncate swap table name by one character.
@@ -766,7 +805,8 @@ public abstract class AbstractJdbcOutputPlugin
766
805
  case INSERT:
767
806
  // aggregate insert into target
768
807
  if (task.getNewTableSchema().isPresent()) {
769
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
808
+ con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
809
+ task.getCreateTableConstraint(), task.getCreateTableOption());
770
810
  }
771
811
  con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), false, task.getBeforeLoad(), task.getAfterLoad());
772
812
  break;
@@ -774,7 +814,8 @@ public abstract class AbstractJdbcOutputPlugin
774
814
  case TRUNCATE_INSERT:
775
815
  // truncate & aggregate insert into target
776
816
  if (task.getNewTableSchema().isPresent()) {
777
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
817
+ con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
818
+ task.getCreateTableConstraint(), task.getCreateTableOption());
778
819
  }
779
820
  con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), true, task.getBeforeLoad(), task.getAfterLoad());
780
821
  break;
@@ -782,7 +823,8 @@ public abstract class AbstractJdbcOutputPlugin
782
823
  case MERGE:
783
824
  // aggregate merge into target
784
825
  if (task.getNewTableSchema().isPresent()) {
785
- con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get());
826
+ con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
827
+ task.getCreateTableConstraint(), task.getCreateTableOption());
786
828
  }
787
829
  con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(),
788
830
  new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()), task.getBeforeLoad(), task.getAfterLoad());
@@ -987,6 +1029,29 @@ public abstract class AbstractJdbcOutputPlugin
987
1029
  }
988
1030
  }
989
1031
 
1032
+ public static File findPluginRoot(Class<?> cls)
1033
+ {
1034
+ try {
1035
+ URL url = cls.getResource("/" + cls.getName().replace('.', '/') + ".class");
1036
+ if (url.toString().startsWith("jar:")) {
1037
+ url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
1038
+ }
1039
+
1040
+ File folder = new File(url.toURI()).getParentFile();
1041
+ for (;; folder = folder.getParentFile()) {
1042
+ if (folder == null) {
1043
+ throw new RuntimeException("Cannot find 'embulk-output-xxx' folder.");
1044
+ }
1045
+
1046
+ if (folder.getName().startsWith("embulk-output-")) {
1047
+ return folder;
1048
+ }
1049
+ }
1050
+ } catch (MalformedURLException | URISyntaxException e) {
1051
+ throw new RuntimeException(e);
1052
+ }
1053
+ }
1054
+
990
1055
  public class PluginPageOutput
991
1056
  implements TransactionalPageOutput
992
1057
  {
@@ -85,6 +85,11 @@ public class JdbcOutputConnection
85
85
  return tableExists(new TableIdentifier(null, schemaName, tableName));
86
86
  }
87
87
 
88
+ protected boolean supportsTableIfExistsClause()
89
+ {
90
+ return true;
91
+ }
92
+
88
93
  public void dropTableIfExists(TableIdentifier table) throws SQLException
89
94
  {
90
95
  Statement stmt = connection.createStatement();
@@ -100,8 +105,14 @@ public class JdbcOutputConnection
100
105
 
101
106
  protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
102
107
  {
103
- String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
104
- executeUpdate(stmt, sql);
108
+ if (supportsTableIfExistsClause()) {
109
+ String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
110
+ executeUpdate(stmt, sql);
111
+ } else {
112
+ if (tableExists(table)) {
113
+ dropTable(stmt, table);
114
+ }
115
+ }
105
116
  }
106
117
 
107
118
  public void dropTable(TableIdentifier table) throws SQLException
@@ -123,35 +134,48 @@ public class JdbcOutputConnection
123
134
  executeUpdate(stmt, sql);
124
135
  }
125
136
 
126
- public void createTableIfNotExists(TableIdentifier targetTable, JdbcSchema schema) throws SQLException
137
+ public void createTableIfNotExists(TableIdentifier table, JdbcSchema schema,
138
+ Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
127
139
  {
128
- Statement stmt = connection.createStatement();
129
- try {
130
- String sql = buildCreateTableIfNotExistsSql(targetTable, schema);
131
- executeUpdate(stmt, sql);
132
- commitIfNecessary(connection);
133
- } catch (SQLException ex) {
134
- throw safeRollback(connection, ex);
135
- } finally {
136
- stmt.close();
140
+ if (supportsTableIfExistsClause()) {
141
+ Statement stmt = connection.createStatement();
142
+ try {
143
+ String sql = buildCreateTableIfNotExistsSql(table, schema, tableConstraint, tableOption);
144
+ executeUpdate(stmt, sql);
145
+ commitIfNecessary(connection);
146
+ } catch (SQLException ex) {
147
+ throw safeRollback(connection, ex);
148
+ } finally {
149
+ stmt.close();
150
+ }
151
+ } else {
152
+ if (!tableExists(table)) {
153
+ createTable(table, schema, tableConstraint, tableOption);
154
+ }
137
155
  }
138
156
  }
139
157
 
140
- protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema)
158
+ protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema,
159
+ Optional<String> tableConstraint, Optional<String> tableOption)
141
160
  {
142
161
  StringBuilder sb = new StringBuilder();
143
162
 
144
163
  sb.append("CREATE TABLE IF NOT EXISTS ");
145
164
  quoteTableIdentifier(sb, table);
146
- sb.append(buildCreateTableSchemaSql(schema));
165
+ sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
166
+ if (tableOption.isPresent()) {
167
+ sb.append(" ");
168
+ sb.append(tableOption.get());
169
+ }
147
170
  return sb.toString();
148
171
  }
149
172
 
150
- public void createTable(TableIdentifier table, JdbcSchema schema) throws SQLException
173
+ public void createTable(TableIdentifier table, JdbcSchema schema,
174
+ Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
151
175
  {
152
176
  Statement stmt = connection.createStatement();
153
177
  try {
154
- String sql = buildCreateTableSql(table, schema);
178
+ String sql = buildCreateTableSql(table, schema, tableConstraint, tableOption);
155
179
  executeUpdate(stmt, sql);
156
180
  commitIfNecessary(connection);
157
181
  } catch (SQLException ex) {
@@ -161,28 +185,37 @@ public class JdbcOutputConnection
161
185
  }
162
186
  }
163
187
 
164
- protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema)
188
+ protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema,
189
+ Optional<String> tableConstraint, Optional<String> tableOption)
165
190
  {
166
191
  StringBuilder sb = new StringBuilder();
167
192
 
168
193
  sb.append("CREATE TABLE ");
169
194
  quoteTableIdentifier(sb, table);
170
- sb.append(buildCreateTableSchemaSql(schema));
195
+ sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
196
+ if (tableOption.isPresent()) {
197
+ sb.append(" ");
198
+ sb.append(tableOption.get());
199
+ }
171
200
  return sb.toString();
172
201
  }
173
202
 
174
- protected String buildCreateTableSchemaSql(JdbcSchema schema)
203
+ protected String buildCreateTableSchemaSql(JdbcSchema schema, Optional<String> tableConstraint)
175
204
  {
176
205
  StringBuilder sb = new StringBuilder();
177
206
 
178
207
  sb.append(" (");
179
- for (int i=0; i < schema.getCount(); i++) {
208
+ for (int i = 0; i < schema.getCount(); i++) {
180
209
  if (i != 0) { sb.append(", "); }
181
210
  quoteIdentifierString(sb, schema.getColumnName(i));
182
211
  sb.append(" ");
183
212
  String typeName = getCreateTableTypeName(schema.getColumn(i));
184
213
  sb.append(typeName);
185
214
  }
215
+ if (tableConstraint.isPresent()) {
216
+ sb.append(", ");
217
+ sb.append(tableConstraint.get());
218
+ }
186
219
  sb.append(")");
187
220
 
188
221
  return sb.toString();
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.12
4
+ version: 0.7.13
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-11-24 00:00:00.000000000 Z
11
+ date: 2017-12-08 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Inserts or updates records to a table.
14
14
  email:
@@ -19,7 +19,7 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-output-jdbc-0.7.12.jar
22
+ - classpath/embulk-output-jdbc-0.7.13.jar
23
23
  - lib/embulk/output/jdbc.rb
24
24
  - src/main/java/org/embulk/output/JdbcOutputPlugin.java
25
25
  - src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java