embulk-output-jdbc 0.7.12 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -0
- data/classpath/embulk-output-jdbc-0.7.13.jar +0 -0
- data/src/main/java/org/embulk/output/JdbcOutputPlugin.java +1 -1
- data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java +92 -27
- data/src/main/java/org/embulk/output/jdbc/JdbcOutputConnection.java +53 -20
- metadata +3 -3
- data/classpath/embulk-output-jdbc-0.7.12.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 411a4c67839886d900bdf7bb5a47216f365103f4
|
4
|
+
data.tar.gz: 53967bc25669370087fa99085b5975785a093630
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 461685691438c570b0ffebd33954cfb24f086b39cac220ecc7483aa0ed678af5f75240019f5f184f0dd5119180c1147e2eba43e3a6e1fcdb39b1331644502093
|
7
|
+
data.tar.gz: 4ed522ff2ae37f35e95c91122e3a146a24e095f9c74f821ded1ee6110c1affbeafc68cfb79c485c37b6750ba7118d15a2d6f63fd69b68db1b0c302819ba3a893
|
data/README.md
CHANGED
@@ -17,6 +17,8 @@ Generic JDBC output plugin for Embulk loads records to a database using a JDBC d
|
|
17
17
|
- **password**: database login password (string, optional)
|
18
18
|
- **schema**: destination schema name (string, default: use default schema)
|
19
19
|
- **table**: destination table name (string, required)
|
20
|
+
- **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
|
21
|
+
- **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
|
20
22
|
- **options**: extra JDBC properties (hash, default: {})
|
21
23
|
- **retry_limit** max retry count for database operations (integer, default: 12)
|
22
24
|
- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
|
Binary file
|
@@ -67,7 +67,7 @@ public class JdbcOutputPlugin
|
|
67
67
|
GenericPluginTask t = (GenericPluginTask) task;
|
68
68
|
|
69
69
|
if (t.getDriverPath().isPresent()) {
|
70
|
-
|
70
|
+
addDriverJarToClasspath(t.getDriverPath().get());
|
71
71
|
}
|
72
72
|
|
73
73
|
Properties props = new Properties();
|
@@ -1,13 +1,18 @@
|
|
1
1
|
package org.embulk.output.jdbc;
|
2
2
|
|
3
|
-
import java.util.HashSet;
|
4
3
|
import java.util.List;
|
5
4
|
import java.util.Map;
|
6
5
|
import java.util.Locale;
|
7
6
|
import java.util.Set;
|
8
7
|
import java.util.concurrent.ExecutionException;
|
8
|
+
import java.io.File;
|
9
|
+
import java.io.FileFilter;
|
9
10
|
import java.io.IOException;
|
11
|
+
import java.net.MalformedURLException;
|
12
|
+
import java.net.URISyntaxException;
|
13
|
+
import java.net.URL;
|
10
14
|
import java.nio.charset.Charset;
|
15
|
+
import java.nio.file.Path;
|
11
16
|
import java.nio.file.Paths;
|
12
17
|
import java.sql.Types;
|
13
18
|
import java.sql.ResultSet;
|
@@ -47,7 +52,6 @@ import org.embulk.spi.Schema;
|
|
47
52
|
import org.embulk.spi.TransactionalPageOutput;
|
48
53
|
import org.embulk.spi.Page;
|
49
54
|
import org.embulk.spi.PageReader;
|
50
|
-
import org.embulk.spi.time.Timestamp;
|
51
55
|
import org.embulk.output.jdbc.setter.ColumnSetter;
|
52
56
|
import org.embulk.output.jdbc.setter.ColumnSetterFactory;
|
53
57
|
import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
|
@@ -59,8 +63,6 @@ import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
|
|
59
63
|
public abstract class AbstractJdbcOutputPlugin
|
60
64
|
implements OutputPlugin
|
61
65
|
{
|
62
|
-
private final static Set<String> loadedJarGlobs = new HashSet<String>();
|
63
|
-
|
64
66
|
protected final Logger logger = Exec.getLogger(getClass());
|
65
67
|
|
66
68
|
public interface PluginTask
|
@@ -89,6 +91,14 @@ public abstract class AbstractJdbcOutputPlugin
|
|
89
91
|
@ConfigDefault("{}")
|
90
92
|
public Map<String, JdbcColumnOption> getColumnOptions();
|
91
93
|
|
94
|
+
@Config("create_table_constraint")
|
95
|
+
@ConfigDefault("null")
|
96
|
+
public Optional<String> getCreateTableConstraint();
|
97
|
+
|
98
|
+
@Config("create_table_option")
|
99
|
+
@ConfigDefault("null")
|
100
|
+
public Optional<String> getCreateTableOption();
|
101
|
+
|
92
102
|
@Config("default_timezone")
|
93
103
|
@ConfigDefault("\"UTC\"")
|
94
104
|
public DateTimeZone getDefaultTimeZone();
|
@@ -217,16 +227,52 @@ public abstract class AbstractJdbcOutputPlugin
|
|
217
227
|
}
|
218
228
|
}
|
219
229
|
|
220
|
-
protected void
|
230
|
+
protected void addDriverJarToClasspath(String glob)
|
221
231
|
{
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
232
|
+
// TODO match glob
|
233
|
+
PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
|
234
|
+
Path path = Paths.get(glob);
|
235
|
+
if (!path.toFile().exists()) {
|
236
|
+
throw new ConfigException("The specified driver jar doesn't exist: " + glob);
|
237
|
+
}
|
238
|
+
loader.addPath(Paths.get(glob));
|
239
|
+
}
|
240
|
+
|
241
|
+
protected void loadDriver(String className, Optional<String> driverPath)
|
242
|
+
{
|
243
|
+
if (driverPath.isPresent()) {
|
244
|
+
addDriverJarToClasspath(driverPath.get());
|
245
|
+
} else {
|
246
|
+
try {
|
247
|
+
// Gradle test task will add JDBC driver to classpath
|
248
|
+
Class.forName(className);
|
249
|
+
|
250
|
+
} catch (ClassNotFoundException ex) {
|
251
|
+
File root = findPluginRoot(getClass());
|
252
|
+
File driverLib = new File(root, "default_jdbc_driver");
|
253
|
+
File[] files = driverLib.listFiles(new FileFilter() {
|
254
|
+
@Override
|
255
|
+
public boolean accept(File file) {
|
256
|
+
return file.isFile() && file.getName().endsWith(".jar");
|
257
|
+
}
|
258
|
+
});
|
259
|
+
if (files == null || files.length == 0) {
|
260
|
+
throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
|
261
|
+
} else {
|
262
|
+
for (File file : files) {
|
263
|
+
logger.info("JDBC Driver = " + file.getAbsolutePath());
|
264
|
+
addDriverJarToClasspath(file.getAbsolutePath());
|
265
|
+
}
|
266
|
+
}
|
228
267
|
}
|
229
268
|
}
|
269
|
+
|
270
|
+
// Load JDBC Driver
|
271
|
+
try {
|
272
|
+
Class.forName(className);
|
273
|
+
} catch (ClassNotFoundException ex) {
|
274
|
+
throw new RuntimeException(ex);
|
275
|
+
}
|
230
276
|
}
|
231
277
|
|
232
278
|
// for subclasses to add @Config
|
@@ -372,13 +418,6 @@ public abstract class AbstractJdbcOutputPlugin
|
|
372
418
|
return commit(task, schema, taskCount);
|
373
419
|
}
|
374
420
|
|
375
|
-
protected String getTransactionUniqueName()
|
376
|
-
{
|
377
|
-
// TODO use uuid?
|
378
|
-
Timestamp t = Exec.session().getTransactionTime();
|
379
|
-
return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
|
380
|
-
}
|
381
|
-
|
382
421
|
private PluginTask begin(final PluginTask task,
|
383
422
|
final Schema schema, final int taskCount)
|
384
423
|
{
|
@@ -525,7 +564,7 @@ public abstract class AbstractJdbcOutputPlugin
|
|
525
564
|
} else {
|
526
565
|
// also create the target table if not exists
|
527
566
|
// CREATE TABLE IF NOT EXISTS xyz
|
528
|
-
con.createTableIfNotExists(task.getActualTable(), newTableSchema);
|
567
|
+
con.createTableIfNotExists(task.getActualTable(), newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
|
529
568
|
targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getActualTable()).get();
|
530
569
|
task.setNewTableSchema(Optional.<JdbcSchema>absent());
|
531
570
|
}
|
@@ -598,17 +637,17 @@ public abstract class AbstractJdbcOutputPlugin
|
|
598
637
|
String namePrefix = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 3,
|
599
638
|
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
600
639
|
for (int taskIndex = 0; taskIndex < taskCount; taskIndex++) {
|
601
|
-
String tableName = namePrefix + String.format("%03d", taskIndex);
|
640
|
+
String tableName = namePrefix + String.format("%03d", taskIndex % 1000);
|
602
641
|
table = buildIntermediateTableId(con, task, tableName);
|
603
642
|
// if table already exists, SQLException will be thrown
|
604
|
-
con.createTable(table, newTableSchema);
|
643
|
+
con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
|
605
644
|
intermTables.add(table);
|
606
645
|
}
|
607
646
|
} else {
|
608
647
|
String tableName = generateIntermediateTableNamePrefix(task.getActualTable().getTableName(), con, 0,
|
609
648
|
task.getFeatures().getMaxTableNameLength(), task.getFeatures().getTableNameLengthSemantics());
|
610
649
|
table = buildIntermediateTableId(con, task, tableName);
|
611
|
-
con.createTable(table, newTableSchema);
|
650
|
+
con.createTable(table, newTableSchema, task.getCreateTableConstraint(), task.getCreateTableOption());
|
612
651
|
intermTables.add(table);
|
613
652
|
}
|
614
653
|
return intermTables.build();
|
@@ -667,8 +706,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
667
706
|
{
|
668
707
|
Charset tableNameCharset = con.getTableNameCharset();
|
669
708
|
String tableName = baseTableName;
|
670
|
-
String suffix = "
|
671
|
-
String uniqueSuffix =
|
709
|
+
String suffix = "_embulk";
|
710
|
+
String uniqueSuffix = String.format("%016x", System.currentTimeMillis()) + suffix;
|
672
711
|
|
673
712
|
// way to count length of table name varies by DBMSs (bytes or characters),
|
674
713
|
// so truncate swap table name by one character.
|
@@ -766,7 +805,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
766
805
|
case INSERT:
|
767
806
|
// aggregate insert into target
|
768
807
|
if (task.getNewTableSchema().isPresent()) {
|
769
|
-
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get()
|
808
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
|
809
|
+
task.getCreateTableConstraint(), task.getCreateTableOption());
|
770
810
|
}
|
771
811
|
con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), false, task.getBeforeLoad(), task.getAfterLoad());
|
772
812
|
break;
|
@@ -774,7 +814,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
774
814
|
case TRUNCATE_INSERT:
|
775
815
|
// truncate & aggregate insert into target
|
776
816
|
if (task.getNewTableSchema().isPresent()) {
|
777
|
-
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get()
|
817
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
|
818
|
+
task.getCreateTableConstraint(), task.getCreateTableOption());
|
778
819
|
}
|
779
820
|
con.collectInsert(task.getIntermediateTables().get(), schema, task.getActualTable(), true, task.getBeforeLoad(), task.getAfterLoad());
|
780
821
|
break;
|
@@ -782,7 +823,8 @@ public abstract class AbstractJdbcOutputPlugin
|
|
782
823
|
case MERGE:
|
783
824
|
// aggregate merge into target
|
784
825
|
if (task.getNewTableSchema().isPresent()) {
|
785
|
-
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get()
|
826
|
+
con.createTableIfNotExists(task.getActualTable(), task.getNewTableSchema().get(),
|
827
|
+
task.getCreateTableConstraint(), task.getCreateTableOption());
|
786
828
|
}
|
787
829
|
con.collectMerge(task.getIntermediateTables().get(), schema, task.getActualTable(),
|
788
830
|
new MergeConfig(task.getMergeKeys().get(), task.getMergeRule()), task.getBeforeLoad(), task.getAfterLoad());
|
@@ -987,6 +1029,29 @@ public abstract class AbstractJdbcOutputPlugin
|
|
987
1029
|
}
|
988
1030
|
}
|
989
1031
|
|
1032
|
+
public static File findPluginRoot(Class<?> cls)
|
1033
|
+
{
|
1034
|
+
try {
|
1035
|
+
URL url = cls.getResource("/" + cls.getName().replace('.', '/') + ".class");
|
1036
|
+
if (url.toString().startsWith("jar:")) {
|
1037
|
+
url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
|
1038
|
+
}
|
1039
|
+
|
1040
|
+
File folder = new File(url.toURI()).getParentFile();
|
1041
|
+
for (;; folder = folder.getParentFile()) {
|
1042
|
+
if (folder == null) {
|
1043
|
+
throw new RuntimeException("Cannot find 'embulk-output-xxx' folder.");
|
1044
|
+
}
|
1045
|
+
|
1046
|
+
if (folder.getName().startsWith("embulk-output-")) {
|
1047
|
+
return folder;
|
1048
|
+
}
|
1049
|
+
}
|
1050
|
+
} catch (MalformedURLException | URISyntaxException e) {
|
1051
|
+
throw new RuntimeException(e);
|
1052
|
+
}
|
1053
|
+
}
|
1054
|
+
|
990
1055
|
public class PluginPageOutput
|
991
1056
|
implements TransactionalPageOutput
|
992
1057
|
{
|
@@ -85,6 +85,11 @@ public class JdbcOutputConnection
|
|
85
85
|
return tableExists(new TableIdentifier(null, schemaName, tableName));
|
86
86
|
}
|
87
87
|
|
88
|
+
protected boolean supportsTableIfExistsClause()
|
89
|
+
{
|
90
|
+
return true;
|
91
|
+
}
|
92
|
+
|
88
93
|
public void dropTableIfExists(TableIdentifier table) throws SQLException
|
89
94
|
{
|
90
95
|
Statement stmt = connection.createStatement();
|
@@ -100,8 +105,14 @@ public class JdbcOutputConnection
|
|
100
105
|
|
101
106
|
protected void dropTableIfExists(Statement stmt, TableIdentifier table) throws SQLException
|
102
107
|
{
|
103
|
-
|
104
|
-
|
108
|
+
if (supportsTableIfExistsClause()) {
|
109
|
+
String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
|
110
|
+
executeUpdate(stmt, sql);
|
111
|
+
} else {
|
112
|
+
if (tableExists(table)) {
|
113
|
+
dropTable(stmt, table);
|
114
|
+
}
|
115
|
+
}
|
105
116
|
}
|
106
117
|
|
107
118
|
public void dropTable(TableIdentifier table) throws SQLException
|
@@ -123,35 +134,48 @@ public class JdbcOutputConnection
|
|
123
134
|
executeUpdate(stmt, sql);
|
124
135
|
}
|
125
136
|
|
126
|
-
public void createTableIfNotExists(TableIdentifier
|
137
|
+
public void createTableIfNotExists(TableIdentifier table, JdbcSchema schema,
|
138
|
+
Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
|
127
139
|
{
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
140
|
+
if (supportsTableIfExistsClause()) {
|
141
|
+
Statement stmt = connection.createStatement();
|
142
|
+
try {
|
143
|
+
String sql = buildCreateTableIfNotExistsSql(table, schema, tableConstraint, tableOption);
|
144
|
+
executeUpdate(stmt, sql);
|
145
|
+
commitIfNecessary(connection);
|
146
|
+
} catch (SQLException ex) {
|
147
|
+
throw safeRollback(connection, ex);
|
148
|
+
} finally {
|
149
|
+
stmt.close();
|
150
|
+
}
|
151
|
+
} else {
|
152
|
+
if (!tableExists(table)) {
|
153
|
+
createTable(table, schema, tableConstraint, tableOption);
|
154
|
+
}
|
137
155
|
}
|
138
156
|
}
|
139
157
|
|
140
|
-
protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema
|
158
|
+
protected String buildCreateTableIfNotExistsSql(TableIdentifier table, JdbcSchema schema,
|
159
|
+
Optional<String> tableConstraint, Optional<String> tableOption)
|
141
160
|
{
|
142
161
|
StringBuilder sb = new StringBuilder();
|
143
162
|
|
144
163
|
sb.append("CREATE TABLE IF NOT EXISTS ");
|
145
164
|
quoteTableIdentifier(sb, table);
|
146
|
-
sb.append(buildCreateTableSchemaSql(schema));
|
165
|
+
sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
|
166
|
+
if (tableOption.isPresent()) {
|
167
|
+
sb.append(" ");
|
168
|
+
sb.append(tableOption.get());
|
169
|
+
}
|
147
170
|
return sb.toString();
|
148
171
|
}
|
149
172
|
|
150
|
-
public void createTable(TableIdentifier table, JdbcSchema schema
|
173
|
+
public void createTable(TableIdentifier table, JdbcSchema schema,
|
174
|
+
Optional<String> tableConstraint, Optional<String> tableOption) throws SQLException
|
151
175
|
{
|
152
176
|
Statement stmt = connection.createStatement();
|
153
177
|
try {
|
154
|
-
String sql = buildCreateTableSql(table, schema);
|
178
|
+
String sql = buildCreateTableSql(table, schema, tableConstraint, tableOption);
|
155
179
|
executeUpdate(stmt, sql);
|
156
180
|
commitIfNecessary(connection);
|
157
181
|
} catch (SQLException ex) {
|
@@ -161,28 +185,37 @@ public class JdbcOutputConnection
|
|
161
185
|
}
|
162
186
|
}
|
163
187
|
|
164
|
-
protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema
|
188
|
+
protected String buildCreateTableSql(TableIdentifier table, JdbcSchema schema,
|
189
|
+
Optional<String> tableConstraint, Optional<String> tableOption)
|
165
190
|
{
|
166
191
|
StringBuilder sb = new StringBuilder();
|
167
192
|
|
168
193
|
sb.append("CREATE TABLE ");
|
169
194
|
quoteTableIdentifier(sb, table);
|
170
|
-
sb.append(buildCreateTableSchemaSql(schema));
|
195
|
+
sb.append(buildCreateTableSchemaSql(schema, tableConstraint));
|
196
|
+
if (tableOption.isPresent()) {
|
197
|
+
sb.append(" ");
|
198
|
+
sb.append(tableOption.get());
|
199
|
+
}
|
171
200
|
return sb.toString();
|
172
201
|
}
|
173
202
|
|
174
|
-
protected String buildCreateTableSchemaSql(JdbcSchema schema)
|
203
|
+
protected String buildCreateTableSchemaSql(JdbcSchema schema, Optional<String> tableConstraint)
|
175
204
|
{
|
176
205
|
StringBuilder sb = new StringBuilder();
|
177
206
|
|
178
207
|
sb.append(" (");
|
179
|
-
for (int i=0; i < schema.getCount(); i++) {
|
208
|
+
for (int i = 0; i < schema.getCount(); i++) {
|
180
209
|
if (i != 0) { sb.append(", "); }
|
181
210
|
quoteIdentifierString(sb, schema.getColumnName(i));
|
182
211
|
sb.append(" ");
|
183
212
|
String typeName = getCreateTableTypeName(schema.getColumn(i));
|
184
213
|
sb.append(typeName);
|
185
214
|
}
|
215
|
+
if (tableConstraint.isPresent()) {
|
216
|
+
sb.append(", ");
|
217
|
+
sb.append(tableConstraint.get());
|
218
|
+
}
|
186
219
|
sb.append(")");
|
187
220
|
|
188
221
|
return sb.toString();
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.7.
|
4
|
+
version: 0.7.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-12-08 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -19,7 +19,7 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-output-jdbc-0.7.
|
22
|
+
- classpath/embulk-output-jdbc-0.7.13.jar
|
23
23
|
- lib/embulk/output/jdbc.rb
|
24
24
|
- src/main/java/org/embulk/output/JdbcOutputPlugin.java
|
25
25
|
- src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java
|
Binary file
|