embulk-input-jdbc 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/embulk-input-jdbc-0.8.1.jar +0 -0
- data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java +22 -12
- data/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java +90 -29
- data/src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java +45 -0
- data/src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java +7 -4
- data/src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java +2 -2
- data/src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java +1 -1
- data/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java +2 -2
- data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java +68 -0
- data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java +75 -0
- metadata +6 -5
- data/classpath/embulk-input-jdbc-0.8.0.jar +0 -0
- data/src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java +0 -254
- data/src/test/java/org/embulk/input/tester/EmbulkPluginTester.java +0 -82
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fc2c284d46271db07625cda2fb5834414a2a463b
|
4
|
+
data.tar.gz: 2e12f5a5958e676da87b69d8ba10e01db1b92657
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0a1cde1000f8214b1585514490a23faecf638d45fdd888a3bc0e96e0e823ac515a43074cf2bdb7f9bf809dc6c4ea0b8ea9f06e0dd82f955a1ad5a9405506334d
|
7
|
+
data.tar.gz: 863a35c8ce727ecca91a992d00b59779577daa6601d0a9a5caaf0239ddb3896150dc2610ca9d80859dab1fe38283065428e197fb92723e5620b0d3ab05c63301
|
Binary file
|
@@ -201,9 +201,9 @@ public abstract class AbstractJdbcInputPlugin
|
|
201
201
|
}
|
202
202
|
|
203
203
|
// build SELECT query and gets schema of its result
|
204
|
-
String
|
204
|
+
String rawQuery = getRawQuery(task, con);
|
205
205
|
|
206
|
-
JdbcSchema querySchema = con.getSchemaOfQuery(
|
206
|
+
JdbcSchema querySchema = con.getSchemaOfQuery(rawQuery);
|
207
207
|
task.setQuerySchema(querySchema);
|
208
208
|
// query schema should not change after incremental query
|
209
209
|
|
@@ -232,26 +232,36 @@ public abstract class AbstractJdbcInputPlugin
|
|
232
232
|
List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
|
233
233
|
task.setIncrementalColumnIndexes(incrementalColumnIndexes);
|
234
234
|
|
235
|
+
List<JsonNode> lastRecord;
|
235
236
|
if (task.getLastRecord().isPresent()) {
|
236
|
-
|
237
|
+
lastRecord = task.getLastRecord().get();
|
237
238
|
if (lastRecord.size() != incrementalColumnIndexes.size()) {
|
238
239
|
throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
|
239
240
|
}
|
240
|
-
preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, lastRecord);
|
241
241
|
}
|
242
242
|
else {
|
243
|
-
|
243
|
+
lastRecord = null;
|
244
|
+
}
|
245
|
+
|
246
|
+
if (task.getQuery().isPresent()) {
|
247
|
+
preparedQuery = con.wrapIncrementalQuery(rawQuery, querySchema, incrementalColumnIndexes, lastRecord);
|
248
|
+
}
|
249
|
+
else {
|
250
|
+
preparedQuery = con.rebuildIncrementalQuery(
|
251
|
+
task.getTable().get(), task.getSelect(),
|
252
|
+
task.getWhere(),
|
253
|
+
querySchema, incrementalColumnIndexes, lastRecord);
|
244
254
|
}
|
245
255
|
}
|
246
256
|
else {
|
247
257
|
task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
|
248
|
-
preparedQuery = new PreparedQuery(
|
258
|
+
preparedQuery = new PreparedQuery(rawQuery, ImmutableList.<JdbcLiteral>of());
|
249
259
|
}
|
250
260
|
|
251
261
|
task.setBuiltQuery(preparedQuery);
|
252
262
|
|
253
263
|
// validate column_options
|
254
|
-
newColumnGetters(task, querySchema, null);
|
264
|
+
newColumnGetters(con, task, querySchema, null);
|
255
265
|
|
256
266
|
ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
|
257
267
|
ImmutableList.Builder<Column> columns = ImmutableList.builder();
|
@@ -260,7 +270,7 @@ public abstract class AbstractJdbcInputPlugin
|
|
260
270
|
JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
|
261
271
|
columns.add(new Column(i,
|
262
272
|
column.getName(),
|
263
|
-
factory.newColumnGetter(column, columnOption).getToType()));
|
273
|
+
factory.newColumnGetter(con, task, column, columnOption).getToType()));
|
264
274
|
}
|
265
275
|
return new Schema(columns.build());
|
266
276
|
}
|
@@ -307,7 +317,7 @@ public abstract class AbstractJdbcInputPlugin
|
|
307
317
|
return builder.build();
|
308
318
|
}
|
309
319
|
|
310
|
-
private String
|
320
|
+
private String getRawQuery(PluginTask task, JdbcInputConnection con) throws SQLException
|
311
321
|
{
|
312
322
|
if (task.getQuery().isPresent()) {
|
313
323
|
if (task.getTable().isPresent() || task.getSelect().isPresent() ||
|
@@ -416,7 +426,7 @@ public abstract class AbstractJdbcInputPlugin
|
|
416
426
|
LastRecordStore lastRecordStore = null;
|
417
427
|
|
418
428
|
try (JdbcInputConnection con = newConnection(task)) {
|
419
|
-
List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
|
429
|
+
List<ColumnGetter> getters = newColumnGetters(con, task, querySchema, pageBuilder);
|
420
430
|
try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
|
421
431
|
while (true) {
|
422
432
|
long rows = fetch(cursor, getters, pageBuilder);
|
@@ -464,14 +474,14 @@ public abstract class AbstractJdbcInputPlugin
|
|
464
474
|
return new ColumnGetterFactory(pageBuilder, dateTimeZone);
|
465
475
|
}
|
466
476
|
|
467
|
-
private List<ColumnGetter> newColumnGetters(PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
|
477
|
+
private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
|
468
478
|
throws SQLException
|
469
479
|
{
|
470
480
|
ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
|
471
481
|
ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
|
472
482
|
for (JdbcColumn c : querySchema.getColumns()) {
|
473
483
|
JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
|
474
|
-
getters.add(factory.newColumnGetter(c, columnOption));
|
484
|
+
getters.add(factory.newColumnGetter(con, task, c, columnOption));
|
475
485
|
}
|
476
486
|
return getters.build();
|
477
487
|
}
|
@@ -248,48 +248,111 @@ public class JdbcInputConnection
|
|
248
248
|
return sb.toString();
|
249
249
|
}
|
250
250
|
|
251
|
-
public PreparedQuery
|
251
|
+
public PreparedQuery rebuildIncrementalQuery(String tableName,
|
252
|
+
Optional<String> selectExpression, Optional<String> whereCondition,
|
253
|
+
JdbcSchema querySchema,
|
254
|
+
List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
|
255
|
+
{
|
256
|
+
List<JdbcLiteral> parameters = ImmutableList.of();
|
257
|
+
|
258
|
+
Optional<String> newWhereCondition;
|
259
|
+
if (incrementalValues != null) {
|
260
|
+
StringBuilder sb = new StringBuilder();
|
261
|
+
|
262
|
+
if (whereCondition.isPresent()) {
|
263
|
+
sb.append("(");
|
264
|
+
sb.append(whereCondition.get());
|
265
|
+
sb.append(") AND ");
|
266
|
+
}
|
267
|
+
|
268
|
+
sb.append("(");
|
269
|
+
parameters = buildIncrementalConditionTo(sb,
|
270
|
+
querySchema, incrementalColumnIndexes, incrementalValues);
|
271
|
+
sb.append(")");
|
272
|
+
|
273
|
+
newWhereCondition = Optional.of(sb.toString());
|
274
|
+
}
|
275
|
+
else {
|
276
|
+
newWhereCondition = whereCondition;
|
277
|
+
}
|
278
|
+
|
279
|
+
Optional<String> newOrderByExpression;
|
280
|
+
{
|
281
|
+
StringBuilder sb = new StringBuilder();
|
282
|
+
buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
|
283
|
+
newOrderByExpression = Optional.of(sb.toString());
|
284
|
+
}
|
285
|
+
|
286
|
+
String newQuery = buildSelectQuery(
|
287
|
+
tableName, selectExpression, newWhereCondition,
|
288
|
+
newOrderByExpression);
|
289
|
+
|
290
|
+
return new PreparedQuery(newQuery, parameters);
|
291
|
+
}
|
292
|
+
|
293
|
+
public PreparedQuery wrapIncrementalQuery(String rawQuery, JdbcSchema querySchema,
|
252
294
|
List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
|
253
295
|
{
|
254
296
|
StringBuilder sb = new StringBuilder();
|
255
|
-
|
297
|
+
List<JdbcLiteral> parameters = ImmutableList.of();
|
256
298
|
|
257
299
|
sb.append("SELECT * FROM (");
|
258
300
|
sb.append(truncateStatementDelimiter(rawQuery));
|
259
301
|
sb.append(") embulk_incremental_");
|
302
|
+
|
260
303
|
if (incrementalValues != null) {
|
261
304
|
sb.append(" WHERE ");
|
305
|
+
parameters = buildIncrementalConditionTo(sb,
|
306
|
+
querySchema, incrementalColumnIndexes, incrementalValues);
|
307
|
+
}
|
262
308
|
|
263
|
-
|
264
|
-
|
265
|
-
for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
|
266
|
-
int columnIndex = incrementalColumnIndexes.get(n);
|
267
|
-
JsonNode value = incrementalValues.get(n);
|
268
|
-
leftColumnNames.add(querySchema.getColumnName(columnIndex));
|
269
|
-
rightLiterals.add(new JdbcLiteral(columnIndex, value));
|
270
|
-
}
|
309
|
+
sb.append(" ORDER BY ");
|
310
|
+
buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
|
271
311
|
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
312
|
+
return new PreparedQuery(sb.toString(), parameters);
|
313
|
+
}
|
314
|
+
|
315
|
+
private List<JdbcLiteral> buildIncrementalConditionTo(
|
316
|
+
StringBuilder sb,
|
317
|
+
JdbcSchema querySchema,
|
318
|
+
List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
|
319
|
+
{
|
320
|
+
ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
|
277
321
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
322
|
+
List<String> leftColumnNames = new ArrayList<>();
|
323
|
+
List<JdbcLiteral> rightLiterals = new ArrayList<>();
|
324
|
+
for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
|
325
|
+
int columnIndex = incrementalColumnIndexes.get(n);
|
326
|
+
JsonNode value = incrementalValues.get(n);
|
327
|
+
leftColumnNames.add(querySchema.getColumnName(columnIndex));
|
328
|
+
rightLiterals.add(new JdbcLiteral(columnIndex, value));
|
329
|
+
}
|
330
|
+
|
331
|
+
for (int n = 0; n < leftColumnNames.size(); n++) {
|
332
|
+
if (n > 0) {
|
333
|
+
sb.append(" OR ");
|
289
334
|
}
|
335
|
+
sb.append("(");
|
336
|
+
|
337
|
+
for (int i = 0; i < n; i++) {
|
338
|
+
sb.append(quoteIdentifierString(leftColumnNames.get(i)));
|
339
|
+
sb.append(" = ?");
|
340
|
+
parameters.add(rightLiterals.get(i));
|
341
|
+
sb.append(" AND ");
|
342
|
+
}
|
343
|
+
sb.append(quoteIdentifierString(leftColumnNames.get(n)));
|
344
|
+
sb.append(" > ?");
|
345
|
+
parameters.add(rightLiterals.get(n));
|
346
|
+
|
347
|
+
sb.append(")");
|
290
348
|
}
|
291
|
-
sb.append(" ORDER BY ");
|
292
349
|
|
350
|
+
return parameters.build();
|
351
|
+
}
|
352
|
+
|
353
|
+
private void buildIncrementalOrderTo(StringBuilder sb,
|
354
|
+
JdbcSchema querySchema, List<Integer> incrementalColumnIndexes)
|
355
|
+
{
|
293
356
|
boolean first = true;
|
294
357
|
for (int i : incrementalColumnIndexes) {
|
295
358
|
if (first) {
|
@@ -299,8 +362,6 @@ public class JdbcInputConnection
|
|
299
362
|
}
|
300
363
|
sb.append(quoteIdentifierString(querySchema.getColumnName(i)));
|
301
364
|
}
|
302
|
-
|
303
|
-
return new PreparedQuery(sb.toString(), parameters.build());
|
304
365
|
}
|
305
366
|
|
306
367
|
protected String truncateStatementDelimiter(String rawQuery) throws SQLException
|
@@ -0,0 +1,45 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import java.sql.ResultSet;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.SQLException;
|
6
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
7
|
+
import com.fasterxml.jackson.databind.node.JsonNodeFactory;
|
8
|
+
import org.embulk.spi.Column;
|
9
|
+
import org.embulk.spi.ColumnVisitor;
|
10
|
+
import org.embulk.spi.PageBuilder;
|
11
|
+
import org.embulk.spi.type.Type;
|
12
|
+
import org.embulk.spi.DataException;
|
13
|
+
import static java.util.Locale.ENGLISH;
|
14
|
+
|
15
|
+
public abstract class AbstractIncrementalHandler implements ColumnGetter
|
16
|
+
{
|
17
|
+
protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
|
18
|
+
|
19
|
+
protected ColumnGetter next;
|
20
|
+
|
21
|
+
public AbstractIncrementalHandler(ColumnGetter next)
|
22
|
+
{
|
23
|
+
this.next = next;
|
24
|
+
}
|
25
|
+
|
26
|
+
@Override
|
27
|
+
public void getAndSet(ResultSet from, int fromIndex,
|
28
|
+
Column toColumn) throws SQLException
|
29
|
+
{
|
30
|
+
next.getAndSet(from, fromIndex, toColumn);
|
31
|
+
}
|
32
|
+
|
33
|
+
@Override
|
34
|
+
public Type getToType()
|
35
|
+
{
|
36
|
+
return next.getToType();
|
37
|
+
}
|
38
|
+
|
39
|
+
@Override
|
40
|
+
public abstract JsonNode encodeToJson();
|
41
|
+
|
42
|
+
@Override
|
43
|
+
public abstract void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
|
44
|
+
throws SQLException;
|
45
|
+
}
|
@@ -9,7 +9,7 @@ import org.embulk.spi.type.Type;
|
|
9
9
|
public abstract class AbstractTimestampColumnGetter
|
10
10
|
extends AbstractColumnGetter
|
11
11
|
{
|
12
|
-
|
12
|
+
protected final TimestampFormatter timestampFormatter;
|
13
13
|
protected Timestamp value;
|
14
14
|
|
15
15
|
public AbstractTimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter)
|
@@ -6,8 +6,10 @@ import java.util.HashMap;
|
|
6
6
|
import java.util.Map;
|
7
7
|
|
8
8
|
import org.embulk.config.ConfigException;
|
9
|
+
import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
|
9
10
|
import org.embulk.input.jdbc.JdbcColumn;
|
10
11
|
import org.embulk.input.jdbc.JdbcColumnOption;
|
12
|
+
import org.embulk.input.jdbc.JdbcInputConnection;
|
11
13
|
import org.embulk.spi.PageBuilder;
|
12
14
|
import org.embulk.spi.time.TimestampFormatter;
|
13
15
|
import org.embulk.spi.type.TimestampType;
|
@@ -28,17 +30,18 @@ public class ColumnGetterFactory
|
|
28
30
|
this.defaultTimeZone = defaultTimeZone;
|
29
31
|
}
|
30
32
|
|
31
|
-
public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
|
33
|
+
public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
|
32
34
|
{
|
33
|
-
return newColumnGetter(column, option, option.getValueType());
|
35
|
+
return newColumnGetter(con, task, column, option, option.getValueType());
|
34
36
|
}
|
35
37
|
|
36
|
-
private ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option, String valueType)
|
38
|
+
private ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option, String valueType)
|
37
39
|
{
|
38
40
|
Type toType = getToType(option);
|
39
41
|
switch(valueType) {
|
40
42
|
case "coalesce":
|
41
|
-
|
43
|
+
// resolve actual valueType using sqlTypeToValueType() method and retry.
|
44
|
+
return newColumnGetter(con, task, column, option, sqlTypeToValueType(column, column.getSqlType()));
|
42
45
|
case "long":
|
43
46
|
return new LongColumnGetter(to, toType);
|
44
47
|
case "float":
|
@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
|
|
15
15
|
public class JsonColumnGetter
|
16
16
|
extends AbstractColumnGetter
|
17
17
|
{
|
18
|
-
final JsonParser jsonParser = new JsonParser();
|
18
|
+
protected final JsonParser jsonParser = new JsonParser();
|
19
19
|
|
20
|
-
|
20
|
+
protected String value;
|
21
21
|
|
22
22
|
public JsonColumnGetter(PageBuilder to, Type toType)
|
23
23
|
{
|
@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
|
|
15
15
|
public class StringColumnGetter
|
16
16
|
extends AbstractColumnGetter
|
17
17
|
{
|
18
|
-
final JsonParser jsonParser = new JsonParser();
|
18
|
+
protected final JsonParser jsonParser = new JsonParser();
|
19
19
|
|
20
|
-
|
20
|
+
protected String value;
|
21
21
|
|
22
22
|
public StringColumnGetter(PageBuilder to, Type toType)
|
23
23
|
{
|
@@ -0,0 +1,68 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.ResultSet;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import java.sql.Timestamp;
|
8
|
+
import org.embulk.spi.Column;
|
9
|
+
import org.embulk.spi.Exec;
|
10
|
+
import org.embulk.spi.time.TimestampFormatter.FormatterTask;
|
11
|
+
import org.embulk.spi.time.TimestampFormatter;
|
12
|
+
import org.embulk.spi.time.TimestampParser.ParserTask;
|
13
|
+
import org.embulk.spi.time.TimestampParser;
|
14
|
+
|
15
|
+
public class TimestampWithTimeZoneIncrementalHandler
|
16
|
+
extends AbstractIncrementalHandler
|
17
|
+
{
|
18
|
+
private static final String ISO_USEC_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ";
|
19
|
+
private static final String ISO_USEC_PATTERN = "%Y-%m-%dT%H:%M:%S.%N%z";
|
20
|
+
|
21
|
+
private long epochSecond;
|
22
|
+
private int nano;
|
23
|
+
|
24
|
+
public TimestampWithTimeZoneIncrementalHandler(ColumnGetter next)
|
25
|
+
{
|
26
|
+
super(next);
|
27
|
+
}
|
28
|
+
|
29
|
+
@Override
|
30
|
+
public void getAndSet(ResultSet from, int fromIndex,
|
31
|
+
Column toColumn) throws SQLException
|
32
|
+
{
|
33
|
+
// sniff the value
|
34
|
+
Timestamp timestamp = from.getTimestamp(fromIndex);
|
35
|
+
if (timestamp != null) {
|
36
|
+
epochSecond = timestamp.getTime() / 1000;
|
37
|
+
nano = timestamp.getNanos();
|
38
|
+
}
|
39
|
+
|
40
|
+
super.getAndSet(from, fromIndex, toColumn);
|
41
|
+
}
|
42
|
+
|
43
|
+
@Override
|
44
|
+
public JsonNode encodeToJson()
|
45
|
+
{
|
46
|
+
FormatterTask task = Exec.newConfigSource()
|
47
|
+
.set("timezone", "UTC")
|
48
|
+
.loadConfig(FormatterTask.class);
|
49
|
+
TimestampFormatter formatter = new TimestampFormatter(ISO_USEC_FORMAT, task);
|
50
|
+
String text = formatter.format(org.embulk.spi.time.Timestamp.ofEpochSecond(epochSecond, nano));
|
51
|
+
return jsonNodeFactory.textNode(text);
|
52
|
+
}
|
53
|
+
|
54
|
+
@Override
|
55
|
+
public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
|
56
|
+
throws SQLException
|
57
|
+
{
|
58
|
+
ParserTask task = Exec.newConfigSource()
|
59
|
+
.set("default_timezone", "UTC")
|
60
|
+
.loadConfig(ParserTask.class);
|
61
|
+
TimestampParser parser = new TimestampParser(ISO_USEC_PATTERN, task);
|
62
|
+
org.embulk.spi.time.Timestamp epoch = parser.parse(fromValue.asText());
|
63
|
+
|
64
|
+
Timestamp sqlTimestamp = new Timestamp(epoch.getEpochSecond() * 1000);
|
65
|
+
sqlTimestamp.setNanos(epoch.getNano());
|
66
|
+
toStatement.setTimestamp(toIndex, sqlTimestamp);
|
67
|
+
}
|
68
|
+
}
|
data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
ADDED
@@ -0,0 +1,75 @@
|
|
1
|
+
package org.embulk.input.jdbc.getter;
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.databind.JsonNode;
|
4
|
+
import java.sql.PreparedStatement;
|
5
|
+
import java.sql.ResultSet;
|
6
|
+
import java.sql.SQLException;
|
7
|
+
import java.sql.Timestamp;
|
8
|
+
import java.util.Calendar;
|
9
|
+
import java.util.TimeZone;
|
10
|
+
import java.util.regex.Matcher;
|
11
|
+
import java.util.regex.Pattern;
|
12
|
+
import org.embulk.config.ConfigException;
|
13
|
+
import org.embulk.spi.Column;
|
14
|
+
import static java.util.Locale.ENGLISH;
|
15
|
+
|
16
|
+
public class TimestampWithoutTimeZoneIncrementalHandler
|
17
|
+
extends AbstractIncrementalHandler
|
18
|
+
{
|
19
|
+
private static final String ISO_USEC_FORMAT = "%d-%02d-%02dT%02d:%02d:%02d.%06d";
|
20
|
+
private static final Pattern ISO_USEC_PATTERN = Pattern.compile("(\\d+)-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}).(\\d{6})");
|
21
|
+
|
22
|
+
private Timestamp dateTime;
|
23
|
+
|
24
|
+
public TimestampWithoutTimeZoneIncrementalHandler(ColumnGetter next)
|
25
|
+
{
|
26
|
+
super(next);
|
27
|
+
}
|
28
|
+
|
29
|
+
@Override
|
30
|
+
public void getAndSet(ResultSet from, int fromIndex,
|
31
|
+
Column toColumn) throws SQLException
|
32
|
+
{
|
33
|
+
// sniff the value
|
34
|
+
Timestamp timestamp = from.getTimestamp(fromIndex);
|
35
|
+
if (timestamp != null) {
|
36
|
+
this.dateTime = timestamp;
|
37
|
+
}
|
38
|
+
|
39
|
+
super.getAndSet(from, fromIndex, toColumn);
|
40
|
+
}
|
41
|
+
|
42
|
+
@Override
|
43
|
+
public JsonNode encodeToJson()
|
44
|
+
{
|
45
|
+
String text = String.format(ENGLISH,
|
46
|
+
ISO_USEC_FORMAT,
|
47
|
+
dateTime.getYear() + 1900,
|
48
|
+
dateTime.getMonth() + 1,
|
49
|
+
dateTime.getDate(),
|
50
|
+
dateTime.getHours(),
|
51
|
+
dateTime.getMinutes(),
|
52
|
+
dateTime.getSeconds(),
|
53
|
+
dateTime.getNanos() / 1000);
|
54
|
+
return jsonNodeFactory.textNode(text);
|
55
|
+
}
|
56
|
+
|
57
|
+
@Override
|
58
|
+
public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
|
59
|
+
throws SQLException
|
60
|
+
{
|
61
|
+
Matcher matcher = ISO_USEC_PATTERN.matcher(fromValue.asText());
|
62
|
+
if (!matcher.matches()) {
|
63
|
+
throw new ConfigException("Invalid timestamp without time zone pattern: " + fromValue);
|
64
|
+
}
|
65
|
+
Timestamp sqlDateTime = new Timestamp(
|
66
|
+
Integer.parseInt(matcher.group(1)) - 1900, // year
|
67
|
+
Integer.parseInt(matcher.group(2)) - 1, // month
|
68
|
+
Integer.parseInt(matcher.group(3)), // day
|
69
|
+
Integer.parseInt(matcher.group(4)), // hour
|
70
|
+
Integer.parseInt(matcher.group(5)), // minute
|
71
|
+
Integer.parseInt(matcher.group(6)), // second
|
72
|
+
Integer.parseInt(matcher.group(7)) * 1000); // usec -> nsec
|
73
|
+
toStatement.setTimestamp(toIndex, sqlDateTime);
|
74
|
+
}
|
75
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-jdbc
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.
|
4
|
+
version: 0.8.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-02-10 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Selects records from a table.
|
14
14
|
email:
|
@@ -19,7 +19,7 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- classpath/embulk-input-jdbc-0.8.
|
22
|
+
- classpath/embulk-input-jdbc-0.8.1.jar
|
23
23
|
- lib/embulk/input/jdbc.rb
|
24
24
|
- src/main/java/org/embulk/input/JdbcInputPlugin.java
|
25
25
|
- src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
|
@@ -31,6 +31,7 @@ files:
|
|
31
31
|
- src/main/java/org/embulk/input/jdbc/ToString.java
|
32
32
|
- src/main/java/org/embulk/input/jdbc/ToStringMap.java
|
33
33
|
- src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java
|
34
|
+
- src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java
|
34
35
|
- src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java
|
35
36
|
- src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java
|
36
37
|
- src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java
|
@@ -44,8 +45,8 @@ files:
|
|
44
45
|
- src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
|
45
46
|
- src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
|
46
47
|
- src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
|
47
|
-
- src/
|
48
|
-
- src/
|
48
|
+
- src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java
|
49
|
+
- src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
|
49
50
|
homepage: https://github.com/embulk/embulk-input-jdbc
|
50
51
|
licenses:
|
51
52
|
- Apache 2.0
|
Binary file
|
@@ -1,254 +0,0 @@
|
|
1
|
-
package org.embulk.input;
|
2
|
-
|
3
|
-
import static java.util.Locale.ENGLISH;
|
4
|
-
|
5
|
-
import java.io.File;
|
6
|
-
import java.io.FileInputStream;
|
7
|
-
import java.io.IOException;
|
8
|
-
import java.io.InputStreamReader;
|
9
|
-
import java.net.URISyntaxException;
|
10
|
-
import java.nio.charset.Charset;
|
11
|
-
import java.sql.Connection;
|
12
|
-
import java.sql.ResultSet;
|
13
|
-
import java.sql.SQLException;
|
14
|
-
import java.sql.Statement;
|
15
|
-
import java.util.ArrayList;
|
16
|
-
import java.util.Collections;
|
17
|
-
import java.util.Comparator;
|
18
|
-
import java.util.List;
|
19
|
-
import java.util.Map;
|
20
|
-
import java.util.regex.Matcher;
|
21
|
-
import java.util.regex.Pattern;
|
22
|
-
|
23
|
-
import org.embulk.config.ConfigException;
|
24
|
-
import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
|
25
|
-
import org.embulk.input.tester.EmbulkPluginTester;
|
26
|
-
import org.embulk.input.tester.EmbulkPluginTester.PluginDefinition;
|
27
|
-
import org.yaml.snakeyaml.Yaml;
|
28
|
-
|
29
|
-
import com.google.common.io.Files;
|
30
|
-
|
31
|
-
public abstract class AbstractJdbcInputPluginTest
|
32
|
-
{
|
33
|
-
private static final String CONFIG_FILE_NAME = "tests.yml";
|
34
|
-
|
35
|
-
protected boolean enabled;
|
36
|
-
// TODO:destroy EmbulkPluginTester after test
|
37
|
-
protected EmbulkPluginTester tester = new EmbulkPluginTester();
|
38
|
-
private String pluginName;
|
39
|
-
private Map<String, ?> testConfigurations;
|
40
|
-
|
41
|
-
protected AbstractJdbcInputPluginTest()
|
42
|
-
{
|
43
|
-
try {
|
44
|
-
prepare();
|
45
|
-
} catch (SQLException e) {
|
46
|
-
throw new RuntimeException(e);
|
47
|
-
}
|
48
|
-
}
|
49
|
-
|
50
|
-
protected abstract void prepare() throws SQLException;
|
51
|
-
|
52
|
-
|
53
|
-
private Map<String, ?> getTestConfigs()
|
54
|
-
{
|
55
|
-
if (testConfigurations == null) {
|
56
|
-
for (PluginDefinition pluginDefinition : tester.getPlugins()) {
|
57
|
-
if (AbstractJdbcInputPlugin.class.isAssignableFrom(pluginDefinition.impl)) {
|
58
|
-
pluginName = pluginDefinition.name;
|
59
|
-
break;
|
60
|
-
}
|
61
|
-
}
|
62
|
-
|
63
|
-
Yaml yaml = new Yaml();
|
64
|
-
File configFile = new File(CONFIG_FILE_NAME);
|
65
|
-
if (!configFile.exists()) {
|
66
|
-
configFile = new File("../" + CONFIG_FILE_NAME);
|
67
|
-
if (!configFile.exists()) {
|
68
|
-
throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't exist.",
|
69
|
-
CONFIG_FILE_NAME));
|
70
|
-
}
|
71
|
-
}
|
72
|
-
|
73
|
-
try {
|
74
|
-
InputStreamReader reader = new InputStreamReader(new FileInputStream(configFile), Charset.forName("UTF8"));
|
75
|
-
try {
|
76
|
-
Map<String, ?> allTestConfigs = (Map<String, ?>)yaml.load(reader);
|
77
|
-
if (!allTestConfigs.containsKey(pluginName)) {
|
78
|
-
throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't contain \"%s\" element.",
|
79
|
-
CONFIG_FILE_NAME, pluginName));
|
80
|
-
}
|
81
|
-
testConfigurations = (Map<String, ?>)allTestConfigs.get(pluginName);
|
82
|
-
} finally {
|
83
|
-
reader.close();
|
84
|
-
}
|
85
|
-
} catch (IOException e) {
|
86
|
-
throw new RuntimeException(e);
|
87
|
-
}
|
88
|
-
}
|
89
|
-
return testConfigurations;
|
90
|
-
}
|
91
|
-
|
92
|
-
protected Object getTestConfig(String name, boolean required)
|
93
|
-
{
|
94
|
-
Map<String, ?> testConfigs = getTestConfigs();
|
95
|
-
if (!testConfigs.containsKey(name)) {
|
96
|
-
if (required) {
|
97
|
-
throw new ConfigException(String.format(ENGLISH, "\"%s\" element in \"%s\" doesn't contain \"%s\" element.",
|
98
|
-
pluginName, CONFIG_FILE_NAME, name));
|
99
|
-
}
|
100
|
-
return null;
|
101
|
-
}
|
102
|
-
return testConfigs.get(name);
|
103
|
-
}
|
104
|
-
|
105
|
-
protected Object getTestConfig(String name)
|
106
|
-
{
|
107
|
-
return getTestConfig(name, true);
|
108
|
-
}
|
109
|
-
|
110
|
-
protected String getHost()
|
111
|
-
{
|
112
|
-
return (String)getTestConfig("host");
|
113
|
-
}
|
114
|
-
|
115
|
-
protected int getPort()
|
116
|
-
{
|
117
|
-
return (Integer)getTestConfig("port");
|
118
|
-
}
|
119
|
-
|
120
|
-
protected String getUser()
|
121
|
-
{
|
122
|
-
return (String)getTestConfig("user");
|
123
|
-
}
|
124
|
-
|
125
|
-
protected String getPassword()
|
126
|
-
{
|
127
|
-
return (String)getTestConfig("password");
|
128
|
-
}
|
129
|
-
|
130
|
-
protected String getDatabase()
|
131
|
-
{
|
132
|
-
return (String)getTestConfig("database");
|
133
|
-
}
|
134
|
-
|
135
|
-
protected void dropTable(String table) throws SQLException
|
136
|
-
{
|
137
|
-
String sql = String.format("DROP TABLE %s", table);
|
138
|
-
executeSQL(sql, true);
|
139
|
-
}
|
140
|
-
|
141
|
-
protected List<List<Object>> select(String table) throws SQLException
|
142
|
-
{
|
143
|
-
try (Connection connection = connect()) {
|
144
|
-
try (Statement statement = connection.createStatement()) {
|
145
|
-
List<List<Object>> rows = new ArrayList<List<Object>>();
|
146
|
-
String sql = String.format("SELECT * FROM %s", table);
|
147
|
-
System.out.println(sql);
|
148
|
-
try (ResultSet resultSet = statement.executeQuery(sql)) {
|
149
|
-
while (resultSet.next()) {
|
150
|
-
List<Object> row = new ArrayList<Object>();
|
151
|
-
for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
|
152
|
-
row.add(getValue(resultSet, i));
|
153
|
-
}
|
154
|
-
rows.add(row);
|
155
|
-
}
|
156
|
-
}
|
157
|
-
// cannot sort by CLOB, so sort by Java
|
158
|
-
Collections.sort(rows, new Comparator<List<Object>>() {
|
159
|
-
@Override
|
160
|
-
public int compare(List<Object> o1, List<Object> o2) {
|
161
|
-
return o1.toString().compareTo(o2.toString());
|
162
|
-
}
|
163
|
-
});
|
164
|
-
return rows;
|
165
|
-
}
|
166
|
-
}
|
167
|
-
|
168
|
-
}
|
169
|
-
|
170
|
-
protected Object getValue(ResultSet resultSet, int index) throws SQLException
|
171
|
-
{
|
172
|
-
return resultSet.getObject(index);
|
173
|
-
}
|
174
|
-
|
175
|
-
protected void executeSQL(String sql) throws SQLException
|
176
|
-
{
|
177
|
-
executeSQL(sql, false);
|
178
|
-
}
|
179
|
-
|
180
|
-
protected void executeSQL(String sql, boolean ignoreError) throws SQLException
|
181
|
-
{
|
182
|
-
if (!enabled) {
|
183
|
-
return;
|
184
|
-
}
|
185
|
-
|
186
|
-
try (Connection connection = connect()) {
|
187
|
-
try {
|
188
|
-
connection.setAutoCommit(true);
|
189
|
-
|
190
|
-
try (Statement statement = connection.createStatement()) {
|
191
|
-
System.out.println(String.format("Execute SQL : \"%s\".", sql));
|
192
|
-
statement.execute(sql);
|
193
|
-
}
|
194
|
-
|
195
|
-
} catch (SQLException e) {
|
196
|
-
if (!ignoreError) {
|
197
|
-
throw e;
|
198
|
-
}
|
199
|
-
}
|
200
|
-
}
|
201
|
-
}
|
202
|
-
|
203
|
-
protected void test(String ymlPath) throws Exception
|
204
|
-
{
|
205
|
-
if (!enabled) {
|
206
|
-
return;
|
207
|
-
}
|
208
|
-
|
209
|
-
tester.run(convertYml(ymlPath));
|
210
|
-
}
|
211
|
-
|
212
|
-
protected String convertYml(String ymlName) throws Exception
|
213
|
-
{
|
214
|
-
StringBuilder builder = new StringBuilder();
|
215
|
-
Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
|
216
|
-
for (String line : Files.readLines(convertPath(ymlName), Charset.forName("UTF8"))) {
|
217
|
-
line = convertYmlLine(line);
|
218
|
-
Matcher matcher = pathPrefixPattern.matcher(line);
|
219
|
-
if (matcher.matches()) {
|
220
|
-
int group = 2;
|
221
|
-
builder.append(line.substring(0, matcher.start(group)));
|
222
|
-
builder.append(convertPath(matcher.group(group)).getAbsolutePath());
|
223
|
-
builder.append(line.substring(matcher.end(group)));
|
224
|
-
} else {
|
225
|
-
builder.append(line);
|
226
|
-
}
|
227
|
-
builder.append(System.lineSeparator());
|
228
|
-
}
|
229
|
-
return builder.toString();
|
230
|
-
}
|
231
|
-
|
232
|
-
protected String convertYmlLine(String line)
|
233
|
-
{
|
234
|
-
line = line.replaceAll("#host#", getHost());
|
235
|
-
line = line.replaceAll("#port#", Integer.toString(getPort()));
|
236
|
-
line = line.replaceAll("#database#", getDatabase());
|
237
|
-
line = line.replaceAll("#user#", getUser());
|
238
|
-
line = line.replaceAll("#password#", getPassword());
|
239
|
-
return line;
|
240
|
-
}
|
241
|
-
|
242
|
-
protected File convertPath(String name) throws URISyntaxException
|
243
|
-
{
|
244
|
-
return new File(getClass().getResource(name).toURI());
|
245
|
-
}
|
246
|
-
|
247
|
-
protected List<String> read(String path) throws IOException
|
248
|
-
{
|
249
|
-
return Files.readLines(new File(path), Charset.forName("UTF8"));
|
250
|
-
}
|
251
|
-
|
252
|
-
protected abstract Connection connect() throws SQLException;
|
253
|
-
|
254
|
-
}
|
@@ -1,82 +0,0 @@
|
|
1
|
-
package org.embulk.input.tester;
|
2
|
-
|
3
|
-
import java.util.ArrayList;
|
4
|
-
import java.util.List;
|
5
|
-
|
6
|
-
import org.embulk.EmbulkEmbed;
|
7
|
-
import org.embulk.EmbulkEmbed.Bootstrap;
|
8
|
-
import org.embulk.config.ConfigSource;
|
9
|
-
import org.embulk.plugin.InjectedPluginSource;
|
10
|
-
|
11
|
-
import com.google.inject.Binder;
|
12
|
-
import com.google.inject.Module;
|
13
|
-
|
14
|
-
public class EmbulkPluginTester
|
15
|
-
{
|
16
|
-
public static class PluginDefinition
|
17
|
-
{
|
18
|
-
public final Class<?> iface;
|
19
|
-
public final String name;
|
20
|
-
public final Class<?> impl;
|
21
|
-
|
22
|
-
|
23
|
-
public PluginDefinition(Class<?> iface, String name, Class<?> impl)
|
24
|
-
{
|
25
|
-
this.iface = iface;
|
26
|
-
this.name = name;
|
27
|
-
this.impl = impl;
|
28
|
-
}
|
29
|
-
|
30
|
-
}
|
31
|
-
|
32
|
-
private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
|
33
|
-
|
34
|
-
private EmbulkEmbed embulk;
|
35
|
-
|
36
|
-
public EmbulkPluginTester()
|
37
|
-
{
|
38
|
-
}
|
39
|
-
|
40
|
-
public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
|
41
|
-
{
|
42
|
-
addPlugin(iface, name, impl);
|
43
|
-
}
|
44
|
-
|
45
|
-
public void addPlugin(Class<?> iface, String name, Class<?> impl)
|
46
|
-
{
|
47
|
-
plugins.add(new PluginDefinition(iface, name, impl));
|
48
|
-
}
|
49
|
-
|
50
|
-
public List<PluginDefinition> getPlugins()
|
51
|
-
{
|
52
|
-
return plugins;
|
53
|
-
}
|
54
|
-
|
55
|
-
public void run(String yml) throws Exception
|
56
|
-
{
|
57
|
-
if (embulk == null) {
|
58
|
-
Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
|
59
|
-
bootstrap.addModules(new Module()
|
60
|
-
{
|
61
|
-
@Override
|
62
|
-
public void configure(Binder binder)
|
63
|
-
{
|
64
|
-
for (PluginDefinition plugin : plugins) {
|
65
|
-
InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
|
66
|
-
}
|
67
|
-
}
|
68
|
-
});
|
69
|
-
embulk = bootstrap.initializeCloseable();
|
70
|
-
}
|
71
|
-
ConfigSource config = embulk.newConfigLoader().fromYamlString(yml);
|
72
|
-
embulk.run(config);
|
73
|
-
}
|
74
|
-
|
75
|
-
public void destroy() {
|
76
|
-
if (embulk != null) {
|
77
|
-
embulk.destroy();
|
78
|
-
embulk = null;
|
79
|
-
}
|
80
|
-
}
|
81
|
-
|
82
|
-
}
|