embulk-input-jdbc 0.8.0 → 0.8.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 121506005fab6a020ff1ab8e908f07e49dd7c5fb
4
- data.tar.gz: 85081ddc42add58819ac8b4a0a9a5e8e52c1f129
3
+ metadata.gz: fc2c284d46271db07625cda2fb5834414a2a463b
4
+ data.tar.gz: 2e12f5a5958e676da87b69d8ba10e01db1b92657
5
5
  SHA512:
6
- metadata.gz: 13c70bfde5dcc988607ea3f77dc84a70402b90878c7bf536ef5e5d3a2530e370b5b75f2eac04dabaabbccbbac9180db60c9612bbebfbf0116c5761adcaae22e2
7
- data.tar.gz: 41c9d0e17c2a825c740d2e0467dc3b1582b8f1d8f6649d8f42d5940ef9e138be731a679a4364c5823a13bc2b46d0949fbf623c00c383a109bd2bd110609c3283
6
+ metadata.gz: 0a1cde1000f8214b1585514490a23faecf638d45fdd888a3bc0e96e0e823ac515a43074cf2bdb7f9bf809dc6c4ea0b8ea9f06e0dd82f955a1ad5a9405506334d
7
+ data.tar.gz: 863a35c8ce727ecca91a992d00b59779577daa6601d0a9a5caaf0239ddb3896150dc2610ca9d80859dab1fe38283065428e197fb92723e5620b0d3ab05c63301
@@ -201,9 +201,9 @@ public abstract class AbstractJdbcInputPlugin
201
201
  }
202
202
 
203
203
  // build SELECT query and gets schema of its result
204
- String query = getQuery(task, con);
204
+ String rawQuery = getRawQuery(task, con);
205
205
 
206
- JdbcSchema querySchema = con.getSchemaOfQuery(query);
206
+ JdbcSchema querySchema = con.getSchemaOfQuery(rawQuery);
207
207
  task.setQuerySchema(querySchema);
208
208
  // query schema should not change after incremental query
209
209
 
@@ -232,26 +232,36 @@ public abstract class AbstractJdbcInputPlugin
232
232
  List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
233
233
  task.setIncrementalColumnIndexes(incrementalColumnIndexes);
234
234
 
235
+ List<JsonNode> lastRecord;
235
236
  if (task.getLastRecord().isPresent()) {
236
- List<JsonNode> lastRecord = task.getLastRecord().get();
237
+ lastRecord = task.getLastRecord().get();
237
238
  if (lastRecord.size() != incrementalColumnIndexes.size()) {
238
239
  throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
239
240
  }
240
- preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, lastRecord);
241
241
  }
242
242
  else {
243
- preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, null);
243
+ lastRecord = null;
244
+ }
245
+
246
+ if (task.getQuery().isPresent()) {
247
+ preparedQuery = con.wrapIncrementalQuery(rawQuery, querySchema, incrementalColumnIndexes, lastRecord);
248
+ }
249
+ else {
250
+ preparedQuery = con.rebuildIncrementalQuery(
251
+ task.getTable().get(), task.getSelect(),
252
+ task.getWhere(),
253
+ querySchema, incrementalColumnIndexes, lastRecord);
244
254
  }
245
255
  }
246
256
  else {
247
257
  task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
248
- preparedQuery = new PreparedQuery(query, ImmutableList.<JdbcLiteral>of());
258
+ preparedQuery = new PreparedQuery(rawQuery, ImmutableList.<JdbcLiteral>of());
249
259
  }
250
260
 
251
261
  task.setBuiltQuery(preparedQuery);
252
262
 
253
263
  // validate column_options
254
- newColumnGetters(task, querySchema, null);
264
+ newColumnGetters(con, task, querySchema, null);
255
265
 
256
266
  ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
257
267
  ImmutableList.Builder<Column> columns = ImmutableList.builder();
@@ -260,7 +270,7 @@ public abstract class AbstractJdbcInputPlugin
260
270
  JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
261
271
  columns.add(new Column(i,
262
272
  column.getName(),
263
- factory.newColumnGetter(column, columnOption).getToType()));
273
+ factory.newColumnGetter(con, task, column, columnOption).getToType()));
264
274
  }
265
275
  return new Schema(columns.build());
266
276
  }
@@ -307,7 +317,7 @@ public abstract class AbstractJdbcInputPlugin
307
317
  return builder.build();
308
318
  }
309
319
 
310
- private String getQuery(PluginTask task, JdbcInputConnection con) throws SQLException
320
+ private String getRawQuery(PluginTask task, JdbcInputConnection con) throws SQLException
311
321
  {
312
322
  if (task.getQuery().isPresent()) {
313
323
  if (task.getTable().isPresent() || task.getSelect().isPresent() ||
@@ -416,7 +426,7 @@ public abstract class AbstractJdbcInputPlugin
416
426
  LastRecordStore lastRecordStore = null;
417
427
 
418
428
  try (JdbcInputConnection con = newConnection(task)) {
419
- List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
429
+ List<ColumnGetter> getters = newColumnGetters(con, task, querySchema, pageBuilder);
420
430
  try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
421
431
  while (true) {
422
432
  long rows = fetch(cursor, getters, pageBuilder);
@@ -464,14 +474,14 @@ public abstract class AbstractJdbcInputPlugin
464
474
  return new ColumnGetterFactory(pageBuilder, dateTimeZone);
465
475
  }
466
476
 
467
- private List<ColumnGetter> newColumnGetters(PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
477
+ private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
468
478
  throws SQLException
469
479
  {
470
480
  ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
471
481
  ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
472
482
  for (JdbcColumn c : querySchema.getColumns()) {
473
483
  JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
474
- getters.add(factory.newColumnGetter(c, columnOption));
484
+ getters.add(factory.newColumnGetter(con, task, c, columnOption));
475
485
  }
476
486
  return getters.build();
477
487
  }
@@ -248,48 +248,111 @@ public class JdbcInputConnection
248
248
  return sb.toString();
249
249
  }
250
250
 
251
- public PreparedQuery buildIncrementalQuery(String rawQuery, JdbcSchema querySchema,
251
+ public PreparedQuery rebuildIncrementalQuery(String tableName,
252
+ Optional<String> selectExpression, Optional<String> whereCondition,
253
+ JdbcSchema querySchema,
254
+ List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
255
+ {
256
+ List<JdbcLiteral> parameters = ImmutableList.of();
257
+
258
+ Optional<String> newWhereCondition;
259
+ if (incrementalValues != null) {
260
+ StringBuilder sb = new StringBuilder();
261
+
262
+ if (whereCondition.isPresent()) {
263
+ sb.append("(");
264
+ sb.append(whereCondition.get());
265
+ sb.append(") AND ");
266
+ }
267
+
268
+ sb.append("(");
269
+ parameters = buildIncrementalConditionTo(sb,
270
+ querySchema, incrementalColumnIndexes, incrementalValues);
271
+ sb.append(")");
272
+
273
+ newWhereCondition = Optional.of(sb.toString());
274
+ }
275
+ else {
276
+ newWhereCondition = whereCondition;
277
+ }
278
+
279
+ Optional<String> newOrderByExpression;
280
+ {
281
+ StringBuilder sb = new StringBuilder();
282
+ buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
283
+ newOrderByExpression = Optional.of(sb.toString());
284
+ }
285
+
286
+ String newQuery = buildSelectQuery(
287
+ tableName, selectExpression, newWhereCondition,
288
+ newOrderByExpression);
289
+
290
+ return new PreparedQuery(newQuery, parameters);
291
+ }
292
+
293
+ public PreparedQuery wrapIncrementalQuery(String rawQuery, JdbcSchema querySchema,
252
294
  List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
253
295
  {
254
296
  StringBuilder sb = new StringBuilder();
255
- ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
297
+ List<JdbcLiteral> parameters = ImmutableList.of();
256
298
 
257
299
  sb.append("SELECT * FROM (");
258
300
  sb.append(truncateStatementDelimiter(rawQuery));
259
301
  sb.append(") embulk_incremental_");
302
+
260
303
  if (incrementalValues != null) {
261
304
  sb.append(" WHERE ");
305
+ parameters = buildIncrementalConditionTo(sb,
306
+ querySchema, incrementalColumnIndexes, incrementalValues);
307
+ }
262
308
 
263
- List<String> leftColumnNames = new ArrayList<>();
264
- List<JdbcLiteral> rightLiterals = new ArrayList<>();
265
- for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
266
- int columnIndex = incrementalColumnIndexes.get(n);
267
- JsonNode value = incrementalValues.get(n);
268
- leftColumnNames.add(querySchema.getColumnName(columnIndex));
269
- rightLiterals.add(new JdbcLiteral(columnIndex, value));
270
- }
309
+ sb.append(" ORDER BY ");
310
+ buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
271
311
 
272
- for (int n = 0; n < leftColumnNames.size(); n++) {
273
- if (n > 0) {
274
- sb.append(" OR ");
275
- }
276
- sb.append("(");
312
+ return new PreparedQuery(sb.toString(), parameters);
313
+ }
314
+
315
+ private List<JdbcLiteral> buildIncrementalConditionTo(
316
+ StringBuilder sb,
317
+ JdbcSchema querySchema,
318
+ List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
319
+ {
320
+ ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
277
321
 
278
- for (int i = 0; i < n; i++) {
279
- sb.append(quoteIdentifierString(leftColumnNames.get(i)));
280
- sb.append(" = ?");
281
- parameters.add(rightLiterals.get(i));
282
- sb.append(" AND ");
283
- }
284
- sb.append(quoteIdentifierString(leftColumnNames.get(n)));
285
- sb.append(" > ?");
286
- parameters.add(rightLiterals.get(n));
287
-
288
- sb.append(")");
322
+ List<String> leftColumnNames = new ArrayList<>();
323
+ List<JdbcLiteral> rightLiterals = new ArrayList<>();
324
+ for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
325
+ int columnIndex = incrementalColumnIndexes.get(n);
326
+ JsonNode value = incrementalValues.get(n);
327
+ leftColumnNames.add(querySchema.getColumnName(columnIndex));
328
+ rightLiterals.add(new JdbcLiteral(columnIndex, value));
329
+ }
330
+
331
+ for (int n = 0; n < leftColumnNames.size(); n++) {
332
+ if (n > 0) {
333
+ sb.append(" OR ");
289
334
  }
335
+ sb.append("(");
336
+
337
+ for (int i = 0; i < n; i++) {
338
+ sb.append(quoteIdentifierString(leftColumnNames.get(i)));
339
+ sb.append(" = ?");
340
+ parameters.add(rightLiterals.get(i));
341
+ sb.append(" AND ");
342
+ }
343
+ sb.append(quoteIdentifierString(leftColumnNames.get(n)));
344
+ sb.append(" > ?");
345
+ parameters.add(rightLiterals.get(n));
346
+
347
+ sb.append(")");
290
348
  }
291
- sb.append(" ORDER BY ");
292
349
 
350
+ return parameters.build();
351
+ }
352
+
353
+ private void buildIncrementalOrderTo(StringBuilder sb,
354
+ JdbcSchema querySchema, List<Integer> incrementalColumnIndexes)
355
+ {
293
356
  boolean first = true;
294
357
  for (int i : incrementalColumnIndexes) {
295
358
  if (first) {
@@ -299,8 +362,6 @@ public class JdbcInputConnection
299
362
  }
300
363
  sb.append(quoteIdentifierString(querySchema.getColumnName(i)));
301
364
  }
302
-
303
- return new PreparedQuery(sb.toString(), parameters.build());
304
365
  }
305
366
 
306
367
  protected String truncateStatementDelimiter(String rawQuery) throws SQLException
@@ -0,0 +1,45 @@
1
+ package org.embulk.input.jdbc.getter;
2
+
3
+ import java.sql.ResultSet;
4
+ import java.sql.PreparedStatement;
5
+ import java.sql.SQLException;
6
+ import com.fasterxml.jackson.databind.JsonNode;
7
+ import com.fasterxml.jackson.databind.node.JsonNodeFactory;
8
+ import org.embulk.spi.Column;
9
+ import org.embulk.spi.ColumnVisitor;
10
+ import org.embulk.spi.PageBuilder;
11
+ import org.embulk.spi.type.Type;
12
+ import org.embulk.spi.DataException;
13
+ import static java.util.Locale.ENGLISH;
14
+
15
+ public abstract class AbstractIncrementalHandler implements ColumnGetter
16
+ {
17
+ protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
18
+
19
+ protected ColumnGetter next;
20
+
21
+ public AbstractIncrementalHandler(ColumnGetter next)
22
+ {
23
+ this.next = next;
24
+ }
25
+
26
+ @Override
27
+ public void getAndSet(ResultSet from, int fromIndex,
28
+ Column toColumn) throws SQLException
29
+ {
30
+ next.getAndSet(from, fromIndex, toColumn);
31
+ }
32
+
33
+ @Override
34
+ public Type getToType()
35
+ {
36
+ return next.getToType();
37
+ }
38
+
39
+ @Override
40
+ public abstract JsonNode encodeToJson();
41
+
42
+ @Override
43
+ public abstract void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
44
+ throws SQLException;
45
+ }
@@ -9,7 +9,7 @@ import org.embulk.spi.type.Type;
9
9
  public abstract class AbstractTimestampColumnGetter
10
10
  extends AbstractColumnGetter
11
11
  {
12
- private final TimestampFormatter timestampFormatter;
12
+ protected final TimestampFormatter timestampFormatter;
13
13
  protected Timestamp value;
14
14
 
15
15
  public AbstractTimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter)
@@ -11,7 +11,7 @@ import org.embulk.spi.type.Types;
11
11
  public class BigDecimalColumnGetter
12
12
  extends AbstractColumnGetter
13
13
  {
14
- private BigDecimal value;
14
+ protected BigDecimal value;
15
15
 
16
16
  public BigDecimalColumnGetter(PageBuilder to, Type toType)
17
17
  {
@@ -10,7 +10,7 @@ import org.embulk.spi.type.Types;
10
10
  public class BooleanColumnGetter
11
11
  extends AbstractColumnGetter
12
12
  {
13
- private boolean value;
13
+ protected boolean value;
14
14
 
15
15
  public BooleanColumnGetter(PageBuilder to, Type toType)
16
16
  {
@@ -6,8 +6,10 @@ import java.util.HashMap;
6
6
  import java.util.Map;
7
7
 
8
8
  import org.embulk.config.ConfigException;
9
+ import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
9
10
  import org.embulk.input.jdbc.JdbcColumn;
10
11
  import org.embulk.input.jdbc.JdbcColumnOption;
12
+ import org.embulk.input.jdbc.JdbcInputConnection;
11
13
  import org.embulk.spi.PageBuilder;
12
14
  import org.embulk.spi.time.TimestampFormatter;
13
15
  import org.embulk.spi.type.TimestampType;
@@ -28,17 +30,18 @@ public class ColumnGetterFactory
28
30
  this.defaultTimeZone = defaultTimeZone;
29
31
  }
30
32
 
31
- public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
33
+ public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
32
34
  {
33
- return newColumnGetter(column, option, option.getValueType());
35
+ return newColumnGetter(con, task, column, option, option.getValueType());
34
36
  }
35
37
 
36
- private ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option, String valueType)
38
+ private ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option, String valueType)
37
39
  {
38
40
  Type toType = getToType(option);
39
41
  switch(valueType) {
40
42
  case "coalesce":
41
- return newColumnGetter(column, option, sqlTypeToValueType(column, column.getSqlType()));
43
+ // resolve actual valueType using sqlTypeToValueType() method and retry.
44
+ return newColumnGetter(con, task, column, option, sqlTypeToValueType(column, column.getSqlType()));
42
45
  case "long":
43
46
  return new LongColumnGetter(to, toType);
44
47
  case "float":
@@ -12,7 +12,7 @@ import com.google.common.math.DoubleMath;
12
12
  public class DoubleColumnGetter
13
13
  extends AbstractColumnGetter
14
14
  {
15
- private double value;
15
+ protected double value;
16
16
 
17
17
  public DoubleColumnGetter(PageBuilder to, Type toType)
18
18
  {
@@ -12,7 +12,7 @@ import com.google.common.math.DoubleMath;
12
12
  public class FloatColumnGetter
13
13
  extends AbstractColumnGetter
14
14
  {
15
- private float value;
15
+ protected float value;
16
16
 
17
17
  public FloatColumnGetter(PageBuilder to, Type toType)
18
18
  {
@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
15
15
  public class JsonColumnGetter
16
16
  extends AbstractColumnGetter
17
17
  {
18
- final JsonParser jsonParser = new JsonParser();
18
+ protected final JsonParser jsonParser = new JsonParser();
19
19
 
20
- private String value;
20
+ protected String value;
21
21
 
22
22
  public JsonColumnGetter(PageBuilder to, Type toType)
23
23
  {
@@ -12,7 +12,7 @@ import org.embulk.spi.type.Types;
12
12
  public class LongColumnGetter
13
13
  extends AbstractColumnGetter
14
14
  {
15
- private long value;
15
+ protected long value;
16
16
 
17
17
  public LongColumnGetter(PageBuilder to, Type toType)
18
18
  {
@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
15
15
  public class StringColumnGetter
16
16
  extends AbstractColumnGetter
17
17
  {
18
- final JsonParser jsonParser = new JsonParser();
18
+ protected final JsonParser jsonParser = new JsonParser();
19
19
 
20
- private String value;
20
+ protected String value;
21
21
 
22
22
  public StringColumnGetter(PageBuilder to, Type toType)
23
23
  {
@@ -0,0 +1,68 @@
1
+ package org.embulk.input.jdbc.getter;
2
+
3
+ import com.fasterxml.jackson.databind.JsonNode;
4
+ import java.sql.PreparedStatement;
5
+ import java.sql.ResultSet;
6
+ import java.sql.SQLException;
7
+ import java.sql.Timestamp;
8
+ import org.embulk.spi.Column;
9
+ import org.embulk.spi.Exec;
10
+ import org.embulk.spi.time.TimestampFormatter.FormatterTask;
11
+ import org.embulk.spi.time.TimestampFormatter;
12
+ import org.embulk.spi.time.TimestampParser.ParserTask;
13
+ import org.embulk.spi.time.TimestampParser;
14
+
15
+ public class TimestampWithTimeZoneIncrementalHandler
16
+ extends AbstractIncrementalHandler
17
+ {
18
+ private static final String ISO_USEC_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ";
19
+ private static final String ISO_USEC_PATTERN = "%Y-%m-%dT%H:%M:%S.%N%z";
20
+
21
+ private long epochSecond;
22
+ private int nano;
23
+
24
+ public TimestampWithTimeZoneIncrementalHandler(ColumnGetter next)
25
+ {
26
+ super(next);
27
+ }
28
+
29
+ @Override
30
+ public void getAndSet(ResultSet from, int fromIndex,
31
+ Column toColumn) throws SQLException
32
+ {
33
+ // sniff the value
34
+ Timestamp timestamp = from.getTimestamp(fromIndex);
35
+ if (timestamp != null) {
36
+ epochSecond = timestamp.getTime() / 1000;
37
+ nano = timestamp.getNanos();
38
+ }
39
+
40
+ super.getAndSet(from, fromIndex, toColumn);
41
+ }
42
+
43
+ @Override
44
+ public JsonNode encodeToJson()
45
+ {
46
+ FormatterTask task = Exec.newConfigSource()
47
+ .set("timezone", "UTC")
48
+ .loadConfig(FormatterTask.class);
49
+ TimestampFormatter formatter = new TimestampFormatter(ISO_USEC_FORMAT, task);
50
+ String text = formatter.format(org.embulk.spi.time.Timestamp.ofEpochSecond(epochSecond, nano));
51
+ return jsonNodeFactory.textNode(text);
52
+ }
53
+
54
+ @Override
55
+ public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
56
+ throws SQLException
57
+ {
58
+ ParserTask task = Exec.newConfigSource()
59
+ .set("default_timezone", "UTC")
60
+ .loadConfig(ParserTask.class);
61
+ TimestampParser parser = new TimestampParser(ISO_USEC_PATTERN, task);
62
+ org.embulk.spi.time.Timestamp epoch = parser.parse(fromValue.asText());
63
+
64
+ Timestamp sqlTimestamp = new Timestamp(epoch.getEpochSecond() * 1000);
65
+ sqlTimestamp.setNanos(epoch.getNano());
66
+ toStatement.setTimestamp(toIndex, sqlTimestamp);
67
+ }
68
+ }
@@ -0,0 +1,75 @@
1
+ package org.embulk.input.jdbc.getter;
2
+
3
+ import com.fasterxml.jackson.databind.JsonNode;
4
+ import java.sql.PreparedStatement;
5
+ import java.sql.ResultSet;
6
+ import java.sql.SQLException;
7
+ import java.sql.Timestamp;
8
+ import java.util.Calendar;
9
+ import java.util.TimeZone;
10
+ import java.util.regex.Matcher;
11
+ import java.util.regex.Pattern;
12
+ import org.embulk.config.ConfigException;
13
+ import org.embulk.spi.Column;
14
+ import static java.util.Locale.ENGLISH;
15
+
16
+ public class TimestampWithoutTimeZoneIncrementalHandler
17
+ extends AbstractIncrementalHandler
18
+ {
19
+ private static final String ISO_USEC_FORMAT = "%d-%02d-%02dT%02d:%02d:%02d.%06d";
20
+ private static final Pattern ISO_USEC_PATTERN = Pattern.compile("(\\d+)-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}).(\\d{6})");
21
+
22
+ private Timestamp dateTime;
23
+
24
+ public TimestampWithoutTimeZoneIncrementalHandler(ColumnGetter next)
25
+ {
26
+ super(next);
27
+ }
28
+
29
+ @Override
30
+ public void getAndSet(ResultSet from, int fromIndex,
31
+ Column toColumn) throws SQLException
32
+ {
33
+ // sniff the value
34
+ Timestamp timestamp = from.getTimestamp(fromIndex);
35
+ if (timestamp != null) {
36
+ this.dateTime = timestamp;
37
+ }
38
+
39
+ super.getAndSet(from, fromIndex, toColumn);
40
+ }
41
+
42
+ @Override
43
+ public JsonNode encodeToJson()
44
+ {
45
+ String text = String.format(ENGLISH,
46
+ ISO_USEC_FORMAT,
47
+ dateTime.getYear() + 1900,
48
+ dateTime.getMonth() + 1,
49
+ dateTime.getDate(),
50
+ dateTime.getHours(),
51
+ dateTime.getMinutes(),
52
+ dateTime.getSeconds(),
53
+ dateTime.getNanos() / 1000);
54
+ return jsonNodeFactory.textNode(text);
55
+ }
56
+
57
+ @Override
58
+ public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
59
+ throws SQLException
60
+ {
61
+ Matcher matcher = ISO_USEC_PATTERN.matcher(fromValue.asText());
62
+ if (!matcher.matches()) {
63
+ throw new ConfigException("Invalid timestamp without time zone pattern: " + fromValue);
64
+ }
65
+ Timestamp sqlDateTime = new Timestamp(
66
+ Integer.parseInt(matcher.group(1)) - 1900, // year
67
+ Integer.parseInt(matcher.group(2)) - 1, // month
68
+ Integer.parseInt(matcher.group(3)), // day
69
+ Integer.parseInt(matcher.group(4)), // hour
70
+ Integer.parseInt(matcher.group(5)), // minute
71
+ Integer.parseInt(matcher.group(6)), // second
72
+ Integer.parseInt(matcher.group(7)) * 1000); // usec -> nsec
73
+ toStatement.setTimestamp(toIndex, sqlDateTime);
74
+ }
75
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-jdbc
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.8.0
4
+ version: 0.8.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-13 00:00:00.000000000 Z
11
+ date: 2017-02-10 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Selects records from a table.
14
14
  email:
@@ -19,7 +19,7 @@ extra_rdoc_files: []
19
19
  files:
20
20
  - README.md
21
21
  - build.gradle
22
- - classpath/embulk-input-jdbc-0.8.0.jar
22
+ - classpath/embulk-input-jdbc-0.8.1.jar
23
23
  - lib/embulk/input/jdbc.rb
24
24
  - src/main/java/org/embulk/input/JdbcInputPlugin.java
25
25
  - src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
@@ -31,6 +31,7 @@ files:
31
31
  - src/main/java/org/embulk/input/jdbc/ToString.java
32
32
  - src/main/java/org/embulk/input/jdbc/ToStringMap.java
33
33
  - src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java
34
+ - src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java
34
35
  - src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java
35
36
  - src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java
36
37
  - src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java
@@ -44,8 +45,8 @@ files:
44
45
  - src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
45
46
  - src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
46
47
  - src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
47
- - src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java
48
- - src/test/java/org/embulk/input/tester/EmbulkPluginTester.java
48
+ - src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java
49
+ - src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
49
50
  homepage: https://github.com/embulk/embulk-input-jdbc
50
51
  licenses:
51
52
  - Apache 2.0
Binary file
@@ -1,254 +0,0 @@
1
- package org.embulk.input;
2
-
3
- import static java.util.Locale.ENGLISH;
4
-
5
- import java.io.File;
6
- import java.io.FileInputStream;
7
- import java.io.IOException;
8
- import java.io.InputStreamReader;
9
- import java.net.URISyntaxException;
10
- import java.nio.charset.Charset;
11
- import java.sql.Connection;
12
- import java.sql.ResultSet;
13
- import java.sql.SQLException;
14
- import java.sql.Statement;
15
- import java.util.ArrayList;
16
- import java.util.Collections;
17
- import java.util.Comparator;
18
- import java.util.List;
19
- import java.util.Map;
20
- import java.util.regex.Matcher;
21
- import java.util.regex.Pattern;
22
-
23
- import org.embulk.config.ConfigException;
24
- import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
25
- import org.embulk.input.tester.EmbulkPluginTester;
26
- import org.embulk.input.tester.EmbulkPluginTester.PluginDefinition;
27
- import org.yaml.snakeyaml.Yaml;
28
-
29
- import com.google.common.io.Files;
30
-
31
- public abstract class AbstractJdbcInputPluginTest
32
- {
33
- private static final String CONFIG_FILE_NAME = "tests.yml";
34
-
35
- protected boolean enabled;
36
- // TODO:destroy EmbulkPluginTester after test
37
- protected EmbulkPluginTester tester = new EmbulkPluginTester();
38
- private String pluginName;
39
- private Map<String, ?> testConfigurations;
40
-
41
- protected AbstractJdbcInputPluginTest()
42
- {
43
- try {
44
- prepare();
45
- } catch (SQLException e) {
46
- throw new RuntimeException(e);
47
- }
48
- }
49
-
50
- protected abstract void prepare() throws SQLException;
51
-
52
-
53
- private Map<String, ?> getTestConfigs()
54
- {
55
- if (testConfigurations == null) {
56
- for (PluginDefinition pluginDefinition : tester.getPlugins()) {
57
- if (AbstractJdbcInputPlugin.class.isAssignableFrom(pluginDefinition.impl)) {
58
- pluginName = pluginDefinition.name;
59
- break;
60
- }
61
- }
62
-
63
- Yaml yaml = new Yaml();
64
- File configFile = new File(CONFIG_FILE_NAME);
65
- if (!configFile.exists()) {
66
- configFile = new File("../" + CONFIG_FILE_NAME);
67
- if (!configFile.exists()) {
68
- throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't exist.",
69
- CONFIG_FILE_NAME));
70
- }
71
- }
72
-
73
- try {
74
- InputStreamReader reader = new InputStreamReader(new FileInputStream(configFile), Charset.forName("UTF8"));
75
- try {
76
- Map<String, ?> allTestConfigs = (Map<String, ?>)yaml.load(reader);
77
- if (!allTestConfigs.containsKey(pluginName)) {
78
- throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't contain \"%s\" element.",
79
- CONFIG_FILE_NAME, pluginName));
80
- }
81
- testConfigurations = (Map<String, ?>)allTestConfigs.get(pluginName);
82
- } finally {
83
- reader.close();
84
- }
85
- } catch (IOException e) {
86
- throw new RuntimeException(e);
87
- }
88
- }
89
- return testConfigurations;
90
- }
91
-
92
- protected Object getTestConfig(String name, boolean required)
93
- {
94
- Map<String, ?> testConfigs = getTestConfigs();
95
- if (!testConfigs.containsKey(name)) {
96
- if (required) {
97
- throw new ConfigException(String.format(ENGLISH, "\"%s\" element in \"%s\" doesn't contain \"%s\" element.",
98
- pluginName, CONFIG_FILE_NAME, name));
99
- }
100
- return null;
101
- }
102
- return testConfigs.get(name);
103
- }
104
-
105
- protected Object getTestConfig(String name)
106
- {
107
- return getTestConfig(name, true);
108
- }
109
-
110
- protected String getHost()
111
- {
112
- return (String)getTestConfig("host");
113
- }
114
-
115
- protected int getPort()
116
- {
117
- return (Integer)getTestConfig("port");
118
- }
119
-
120
- protected String getUser()
121
- {
122
- return (String)getTestConfig("user");
123
- }
124
-
125
- protected String getPassword()
126
- {
127
- return (String)getTestConfig("password");
128
- }
129
-
130
- protected String getDatabase()
131
- {
132
- return (String)getTestConfig("database");
133
- }
134
-
135
- protected void dropTable(String table) throws SQLException
136
- {
137
- String sql = String.format("DROP TABLE %s", table);
138
- executeSQL(sql, true);
139
- }
140
-
141
- protected List<List<Object>> select(String table) throws SQLException
142
- {
143
- try (Connection connection = connect()) {
144
- try (Statement statement = connection.createStatement()) {
145
- List<List<Object>> rows = new ArrayList<List<Object>>();
146
- String sql = String.format("SELECT * FROM %s", table);
147
- System.out.println(sql);
148
- try (ResultSet resultSet = statement.executeQuery(sql)) {
149
- while (resultSet.next()) {
150
- List<Object> row = new ArrayList<Object>();
151
- for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
152
- row.add(getValue(resultSet, i));
153
- }
154
- rows.add(row);
155
- }
156
- }
157
- // cannot sort by CLOB, so sort by Java
158
- Collections.sort(rows, new Comparator<List<Object>>() {
159
- @Override
160
- public int compare(List<Object> o1, List<Object> o2) {
161
- return o1.toString().compareTo(o2.toString());
162
- }
163
- });
164
- return rows;
165
- }
166
- }
167
-
168
- }
169
-
170
- protected Object getValue(ResultSet resultSet, int index) throws SQLException
171
- {
172
- return resultSet.getObject(index);
173
- }
174
-
175
- protected void executeSQL(String sql) throws SQLException
176
- {
177
- executeSQL(sql, false);
178
- }
179
-
180
- protected void executeSQL(String sql, boolean ignoreError) throws SQLException
181
- {
182
- if (!enabled) {
183
- return;
184
- }
185
-
186
- try (Connection connection = connect()) {
187
- try {
188
- connection.setAutoCommit(true);
189
-
190
- try (Statement statement = connection.createStatement()) {
191
- System.out.println(String.format("Execute SQL : \"%s\".", sql));
192
- statement.execute(sql);
193
- }
194
-
195
- } catch (SQLException e) {
196
- if (!ignoreError) {
197
- throw e;
198
- }
199
- }
200
- }
201
- }
202
-
203
- protected void test(String ymlPath) throws Exception
204
- {
205
- if (!enabled) {
206
- return;
207
- }
208
-
209
- tester.run(convertYml(ymlPath));
210
- }
211
-
212
- protected String convertYml(String ymlName) throws Exception
213
- {
214
- StringBuilder builder = new StringBuilder();
215
- Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
216
- for (String line : Files.readLines(convertPath(ymlName), Charset.forName("UTF8"))) {
217
- line = convertYmlLine(line);
218
- Matcher matcher = pathPrefixPattern.matcher(line);
219
- if (matcher.matches()) {
220
- int group = 2;
221
- builder.append(line.substring(0, matcher.start(group)));
222
- builder.append(convertPath(matcher.group(group)).getAbsolutePath());
223
- builder.append(line.substring(matcher.end(group)));
224
- } else {
225
- builder.append(line);
226
- }
227
- builder.append(System.lineSeparator());
228
- }
229
- return builder.toString();
230
- }
231
-
232
- protected String convertYmlLine(String line)
233
- {
234
- line = line.replaceAll("#host#", getHost());
235
- line = line.replaceAll("#port#", Integer.toString(getPort()));
236
- line = line.replaceAll("#database#", getDatabase());
237
- line = line.replaceAll("#user#", getUser());
238
- line = line.replaceAll("#password#", getPassword());
239
- return line;
240
- }
241
-
242
- protected File convertPath(String name) throws URISyntaxException
243
- {
244
- return new File(getClass().getResource(name).toURI());
245
- }
246
-
247
- protected List<String> read(String path) throws IOException
248
- {
249
- return Files.readLines(new File(path), Charset.forName("UTF8"));
250
- }
251
-
252
- protected abstract Connection connect() throws SQLException;
253
-
254
- }
@@ -1,82 +0,0 @@
1
- package org.embulk.input.tester;
2
-
3
- import java.util.ArrayList;
4
- import java.util.List;
5
-
6
- import org.embulk.EmbulkEmbed;
7
- import org.embulk.EmbulkEmbed.Bootstrap;
8
- import org.embulk.config.ConfigSource;
9
- import org.embulk.plugin.InjectedPluginSource;
10
-
11
- import com.google.inject.Binder;
12
- import com.google.inject.Module;
13
-
14
- public class EmbulkPluginTester
15
- {
16
- public static class PluginDefinition
17
- {
18
- public final Class<?> iface;
19
- public final String name;
20
- public final Class<?> impl;
21
-
22
-
23
- public PluginDefinition(Class<?> iface, String name, Class<?> impl)
24
- {
25
- this.iface = iface;
26
- this.name = name;
27
- this.impl = impl;
28
- }
29
-
30
- }
31
-
32
- private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
33
-
34
- private EmbulkEmbed embulk;
35
-
36
- public EmbulkPluginTester()
37
- {
38
- }
39
-
40
- public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
41
- {
42
- addPlugin(iface, name, impl);
43
- }
44
-
45
- public void addPlugin(Class<?> iface, String name, Class<?> impl)
46
- {
47
- plugins.add(new PluginDefinition(iface, name, impl));
48
- }
49
-
50
- public List<PluginDefinition> getPlugins()
51
- {
52
- return plugins;
53
- }
54
-
55
- public void run(String yml) throws Exception
56
- {
57
- if (embulk == null) {
58
- Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
59
- bootstrap.addModules(new Module()
60
- {
61
- @Override
62
- public void configure(Binder binder)
63
- {
64
- for (PluginDefinition plugin : plugins) {
65
- InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
66
- }
67
- }
68
- });
69
- embulk = bootstrap.initializeCloseable();
70
- }
71
- ConfigSource config = embulk.newConfigLoader().fromYamlString(yml);
72
- embulk.run(config);
73
- }
74
-
75
- public void destroy() {
76
- if (embulk != null) {
77
- embulk.destroy();
78
- embulk = null;
79
- }
80
- }
81
-
82
- }