embulk-input-jdbc 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 121506005fab6a020ff1ab8e908f07e49dd7c5fb
-  data.tar.gz: 85081ddc42add58819ac8b4a0a9a5e8e52c1f129
+  metadata.gz: fc2c284d46271db07625cda2fb5834414a2a463b
+  data.tar.gz: 2e12f5a5958e676da87b69d8ba10e01db1b92657
 SHA512:
-  metadata.gz: 13c70bfde5dcc988607ea3f77dc84a70402b90878c7bf536ef5e5d3a2530e370b5b75f2eac04dabaabbccbbac9180db60c9612bbebfbf0116c5761adcaae22e2
-  data.tar.gz: 41c9d0e17c2a825c740d2e0467dc3b1582b8f1d8f6649d8f42d5940ef9e138be731a679a4364c5823a13bc2b46d0949fbf623c00c383a109bd2bd110609c3283
+  metadata.gz: 0a1cde1000f8214b1585514490a23faecf638d45fdd888a3bc0e96e0e823ac515a43074cf2bdb7f9bf809dc6c4ea0b8ea9f06e0dd82f955a1ad5a9405506334d
+  data.tar.gz: 863a35c8ce727ecca91a992d00b59779577daa6601d0a9a5caaf0239ddb3896150dc2610ca9d80859dab1fe38283065428e197fb92723e5620b0d3ab05c63301

data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java

@@ -201,9 +201,9 @@ public abstract class AbstractJdbcInputPlugin
         }
 
         // build SELECT query and gets schema of its result
-        String query = getQuery(task, con);
+        String rawQuery = getRawQuery(task, con);
 
-        JdbcSchema querySchema = con.getSchemaOfQuery(query);
+        JdbcSchema querySchema = con.getSchemaOfQuery(rawQuery);
         task.setQuerySchema(querySchema);
         // query schema should not change after incremental query
 
@@ -232,26 +232,36 @@ public abstract class AbstractJdbcInputPlugin
             List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
             task.setIncrementalColumnIndexes(incrementalColumnIndexes);
 
+            List<JsonNode> lastRecord;
             if (task.getLastRecord().isPresent()) {
-                List<JsonNode> lastRecord = task.getLastRecord().get();
+                lastRecord = task.getLastRecord().get();
                 if (lastRecord.size() != incrementalColumnIndexes.size()) {
                     throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
                 }
-                preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, lastRecord);
             }
             else {
-                preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, null);
+                lastRecord = null;
+            }
+
+            if (task.getQuery().isPresent()) {
+                preparedQuery = con.wrapIncrementalQuery(rawQuery, querySchema, incrementalColumnIndexes, lastRecord);
+            }
+            else {
+                preparedQuery = con.rebuildIncrementalQuery(
+                        task.getTable().get(), task.getSelect(),
+                        task.getWhere(),
+                        querySchema, incrementalColumnIndexes, lastRecord);
             }
         }
         else {
             task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
-            preparedQuery = new PreparedQuery(query, ImmutableList.<JdbcLiteral>of());
+            preparedQuery = new PreparedQuery(rawQuery, ImmutableList.<JdbcLiteral>of());
         }
 
         task.setBuiltQuery(preparedQuery);
 
         // validate column_options
-        newColumnGetters(task, querySchema, null);
+        newColumnGetters(con, task, querySchema, null);
 
         ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
         ImmutableList.Builder<Column> columns = ImmutableList.builder();
@@ -260,7 +270,7 @@ public abstract class AbstractJdbcInputPlugin
             JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
             columns.add(new Column(i,
                     column.getName(),
-                    factory.newColumnGetter(column, columnOption).getToType()));
+                    factory.newColumnGetter(con, task, column, columnOption).getToType()));
         }
         return new Schema(columns.build());
     }
@@ -307,7 +317,7 @@ public abstract class AbstractJdbcInputPlugin
         return builder.build();
     }
 
-    private String getQuery(PluginTask task, JdbcInputConnection con) throws SQLException
+    private String getRawQuery(PluginTask task, JdbcInputConnection con) throws SQLException
     {
         if (task.getQuery().isPresent()) {
             if (task.getTable().isPresent() || task.getSelect().isPresent() ||
@@ -416,7 +426,7 @@ public abstract class AbstractJdbcInputPlugin
         LastRecordStore lastRecordStore = null;
 
         try (JdbcInputConnection con = newConnection(task)) {
-            List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
+            List<ColumnGetter> getters = newColumnGetters(con, task, querySchema, pageBuilder);
             try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
                 while (true) {
                     long rows = fetch(cursor, getters, pageBuilder);
@@ -464,14 +474,14 @@ public abstract class AbstractJdbcInputPlugin
         return new ColumnGetterFactory(pageBuilder, dateTimeZone);
     }
 
-    private List<ColumnGetter> newColumnGetters(PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
+    private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
             throws SQLException
     {
         ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
         ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
         for (JdbcColumn c : querySchema.getColumns()) {
             JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
-            getters.add(factory.newColumnGetter(c, columnOption));
+            getters.add(factory.newColumnGetter(con, task, c, columnOption));
         }
         return getters.build();
     }

data/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java

@@ -248,48 +248,111 @@ public class JdbcInputConnection
         return sb.toString();
     }
 
-    public PreparedQuery buildIncrementalQuery(String rawQuery, JdbcSchema querySchema,
+    public PreparedQuery rebuildIncrementalQuery(String tableName,
+            Optional<String> selectExpression, Optional<String> whereCondition,
+            JdbcSchema querySchema,
+            List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
+    {
+        List<JdbcLiteral> parameters = ImmutableList.of();
+
+        Optional<String> newWhereCondition;
+        if (incrementalValues != null) {
+            StringBuilder sb = new StringBuilder();
+
+            if (whereCondition.isPresent()) {
+                sb.append("(");
+                sb.append(whereCondition.get());
+                sb.append(") AND ");
+            }
+
+            sb.append("(");
+            parameters = buildIncrementalConditionTo(sb,
+                    querySchema, incrementalColumnIndexes, incrementalValues);
+            sb.append(")");
+
+            newWhereCondition = Optional.of(sb.toString());
+        }
+        else {
+            newWhereCondition = whereCondition;
+        }
+
+        Optional<String> newOrderByExpression;
+        {
+            StringBuilder sb = new StringBuilder();
+            buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
+            newOrderByExpression = Optional.of(sb.toString());
+        }
+
+        String newQuery = buildSelectQuery(
+                tableName, selectExpression, newWhereCondition,
+                newOrderByExpression);
+
+        return new PreparedQuery(newQuery, parameters);
+    }
+
+    public PreparedQuery wrapIncrementalQuery(String rawQuery, JdbcSchema querySchema,
             List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
-        ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
+        List<JdbcLiteral> parameters = ImmutableList.of();
 
         sb.append("SELECT * FROM (");
         sb.append(truncateStatementDelimiter(rawQuery));
         sb.append(") embulk_incremental_");
+
         if (incrementalValues != null) {
             sb.append(" WHERE ");
+            parameters = buildIncrementalConditionTo(sb,
+                    querySchema, incrementalColumnIndexes, incrementalValues);
+        }
 
-            List<String> leftColumnNames = new ArrayList<>();
-            List<JdbcLiteral> rightLiterals = new ArrayList<>();
-            for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
-                int columnIndex = incrementalColumnIndexes.get(n);
-                JsonNode value = incrementalValues.get(n);
-                leftColumnNames.add(querySchema.getColumnName(columnIndex));
-                rightLiterals.add(new JdbcLiteral(columnIndex, value));
-            }
+        sb.append(" ORDER BY ");
+        buildIncrementalOrderTo(sb, querySchema, incrementalColumnIndexes);
 
-            for (int n = 0; n < leftColumnNames.size(); n++) {
-                if (n > 0) {
-                    sb.append(" OR ");
-                }
-                sb.append("(");
+        return new PreparedQuery(sb.toString(), parameters);
+    }
+
+    private List<JdbcLiteral> buildIncrementalConditionTo(
+            StringBuilder sb,
+            JdbcSchema querySchema,
+            List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
+    {
+        ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
 
-                for (int i = 0; i < n; i++) {
-                    sb.append(quoteIdentifierString(leftColumnNames.get(i)));
-                    sb.append(" = ?");
-                    parameters.add(rightLiterals.get(i));
-                    sb.append(" AND ");
-                }
-                sb.append(quoteIdentifierString(leftColumnNames.get(n)));
-                sb.append(" > ?");
-                parameters.add(rightLiterals.get(n));
-
-                sb.append(")");
+        List<String> leftColumnNames = new ArrayList<>();
+        List<JdbcLiteral> rightLiterals = new ArrayList<>();
+        for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
+            int columnIndex = incrementalColumnIndexes.get(n);
+            JsonNode value = incrementalValues.get(n);
+            leftColumnNames.add(querySchema.getColumnName(columnIndex));
+            rightLiterals.add(new JdbcLiteral(columnIndex, value));
+        }
+
+        for (int n = 0; n < leftColumnNames.size(); n++) {
+            if (n > 0) {
+                sb.append(" OR ");
             }
+            sb.append("(");
+
+            for (int i = 0; i < n; i++) {
+                sb.append(quoteIdentifierString(leftColumnNames.get(i)));
+                sb.append(" = ?");
+                parameters.add(rightLiterals.get(i));
+                sb.append(" AND ");
+            }
+            sb.append(quoteIdentifierString(leftColumnNames.get(n)));
+            sb.append(" > ?");
+            parameters.add(rightLiterals.get(n));
+
+            sb.append(")");
         }
-        sb.append(" ORDER BY ");
 
+        return parameters.build();
+    }
+
+    private void buildIncrementalOrderTo(StringBuilder sb,
+            JdbcSchema querySchema, List<Integer> incrementalColumnIndexes)
+    {
         boolean first = true;
         for (int i : incrementalColumnIndexes) {
             if (first) {
@@ -299,8 +362,6 @@ public class JdbcInputConnection
             }
             sb.append(quoteIdentifierString(querySchema.getColumnName(i)));
         }
-
-        return new PreparedQuery(sb.toString(), parameters.build());
     }
 
     protected String truncateStatementDelimiter(String rawQuery) throws SQLException

data/src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java

@@ -0,0 +1,45 @@
+package org.embulk.input.jdbc.getter;
+
+import java.sql.ResultSet;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.PageBuilder;
+import org.embulk.spi.type.Type;
+import org.embulk.spi.DataException;
+import static java.util.Locale.ENGLISH;
+
+public abstract class AbstractIncrementalHandler implements ColumnGetter
+{
+    protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
+
+    protected ColumnGetter next;
+
+    public AbstractIncrementalHandler(ColumnGetter next)
+    {
+        this.next = next;
+    }
+
+    @Override
+    public void getAndSet(ResultSet from, int fromIndex,
+            Column toColumn) throws SQLException
+    {
+        next.getAndSet(from, fromIndex, toColumn);
+    }
+
+    @Override
+    public Type getToType()
+    {
+        return next.getToType();
+    }
+
+    @Override
+    public abstract JsonNode encodeToJson();
+
+    @Override
+    public abstract void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException;
+}

data/src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java

@@ -9,7 +9,7 @@ import org.embulk.spi.type.Type;
 public abstract class AbstractTimestampColumnGetter
         extends AbstractColumnGetter
 {
-    private final TimestampFormatter timestampFormatter;
+    protected final TimestampFormatter timestampFormatter;
     protected Timestamp value;
 
     public AbstractTimestampColumnGetter(PageBuilder to, Type toType, TimestampFormatter timestampFormatter)

data/src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java

@@ -11,7 +11,7 @@ import org.embulk.spi.type.Types;
 public class BigDecimalColumnGetter
         extends AbstractColumnGetter
 {
-    private BigDecimal value;
+    protected BigDecimal value;
 
     public BigDecimalColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java

@@ -10,7 +10,7 @@ import org.embulk.spi.type.Types;
 public class BooleanColumnGetter
         extends AbstractColumnGetter
 {
-    private boolean value;
+    protected boolean value;
 
     public BooleanColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java

@@ -6,8 +6,10 @@ import java.util.HashMap;
 import java.util.Map;
 
 import org.embulk.config.ConfigException;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
 import org.embulk.input.jdbc.JdbcColumn;
 import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.JdbcInputConnection;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.time.TimestampFormatter;
 import org.embulk.spi.type.TimestampType;
@@ -28,17 +30,18 @@ public class ColumnGetterFactory
         this.defaultTimeZone = defaultTimeZone;
     }
 
-    public ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option)
+    public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
     {
-        return newColumnGetter(column, option, option.getValueType());
+        return newColumnGetter(con, task, column, option, option.getValueType());
     }
 
-    private ColumnGetter newColumnGetter(JdbcColumn column, JdbcColumnOption option, String valueType)
+    private ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option, String valueType)
     {
         Type toType = getToType(option);
         switch(valueType) {
         case "coalesce":
-            return newColumnGetter(column, option, sqlTypeToValueType(column, column.getSqlType()));
+            // resolve actual valueType using sqlTypeToValueType() method and retry.
+            return newColumnGetter(con, task, column, option, sqlTypeToValueType(column, column.getSqlType()));
         case "long":
             return new LongColumnGetter(to, toType);
         case "float":

data/src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java

@@ -12,7 +12,7 @@ import com.google.common.math.DoubleMath;
 public class DoubleColumnGetter
         extends AbstractColumnGetter
 {
-    private double value;
+    protected double value;
 
     public DoubleColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java

@@ -12,7 +12,7 @@ import com.google.common.math.DoubleMath;
 public class FloatColumnGetter
         extends AbstractColumnGetter
 {
-    private float value;
+    protected float value;
 
     public FloatColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java

@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
 public class JsonColumnGetter
         extends AbstractColumnGetter
 {
-    final JsonParser jsonParser = new JsonParser();
+    protected final JsonParser jsonParser = new JsonParser();
 
-    private String value;
+    protected String value;
 
     public JsonColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java

@@ -12,7 +12,7 @@ import org.embulk.spi.type.Types;
 public class LongColumnGetter
         extends AbstractColumnGetter
 {
-    private long value;
+    protected long value;
 
     public LongColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java

@@ -15,9 +15,9 @@ import org.msgpack.value.Value;
 public class StringColumnGetter
         extends AbstractColumnGetter
 {
-    final JsonParser jsonParser = new JsonParser();
+    protected final JsonParser jsonParser = new JsonParser();
 
-    private String value;
+    protected String value;
 
     public StringColumnGetter(PageBuilder to, Type toType)
     {

data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java

@@ -0,0 +1,68 @@
+package org.embulk.input.jdbc.getter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.time.TimestampFormatter.FormatterTask;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampParser.ParserTask;
+import org.embulk.spi.time.TimestampParser;
+
+public class TimestampWithTimeZoneIncrementalHandler
+        extends AbstractIncrementalHandler
+{
+    private static final String ISO_USEC_FORMAT = "%Y-%m-%dT%H:%M:%S.%6NZ";
+    private static final String ISO_USEC_PATTERN = "%Y-%m-%dT%H:%M:%S.%N%z";
+
+    private long epochSecond;
+    private int nano;
+
+    public TimestampWithTimeZoneIncrementalHandler(ColumnGetter next)
+    {
+        super(next);
+    }
+
+    @Override
+    public void getAndSet(ResultSet from, int fromIndex,
+            Column toColumn) throws SQLException
+    {
+        // sniff the value
+        Timestamp timestamp = from.getTimestamp(fromIndex);
+        if (timestamp != null) {
+            epochSecond = timestamp.getTime() / 1000;
+            nano = timestamp.getNanos();
+        }
+
+        super.getAndSet(from, fromIndex, toColumn);
+    }
+
+    @Override
+    public JsonNode encodeToJson()
+    {
+        FormatterTask task = Exec.newConfigSource()
+            .set("timezone", "UTC")
+            .loadConfig(FormatterTask.class);
+        TimestampFormatter formatter = new TimestampFormatter(ISO_USEC_FORMAT, task);
+        String text = formatter.format(org.embulk.spi.time.Timestamp.ofEpochSecond(epochSecond, nano));
+        return jsonNodeFactory.textNode(text);
+    }
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException
+    {
+        ParserTask task = Exec.newConfigSource()
+            .set("default_timezone", "UTC")
+            .loadConfig(ParserTask.class);
+        TimestampParser parser = new TimestampParser(ISO_USEC_PATTERN, task);
+        org.embulk.spi.time.Timestamp epoch = parser.parse(fromValue.asText());
+
+        Timestamp sqlTimestamp = new Timestamp(epoch.getEpochSecond() * 1000);
+        sqlTimestamp.setNanos(epoch.getNano());
+        toStatement.setTimestamp(toIndex, sqlTimestamp);
+    }
+}

data/src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java

@@ -0,0 +1,75 @@
+package org.embulk.input.jdbc.getter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.util.Calendar;
+import java.util.TimeZone;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import org.embulk.config.ConfigException;
+import org.embulk.spi.Column;
+import static java.util.Locale.ENGLISH;
+
+public class TimestampWithoutTimeZoneIncrementalHandler
+        extends AbstractIncrementalHandler
+{
+    private static final String ISO_USEC_FORMAT = "%d-%02d-%02dT%02d:%02d:%02d.%06d";
+    private static final Pattern ISO_USEC_PATTERN = Pattern.compile("(\\d+)-(\\d{2})-(\\d{2})T(\\d{2}):(\\d{2}):(\\d{2}).(\\d{6})");
+
+    private Timestamp dateTime;
+
+    public TimestampWithoutTimeZoneIncrementalHandler(ColumnGetter next)
+    {
+        super(next);
+    }
+
+    @Override
+    public void getAndSet(ResultSet from, int fromIndex,
+            Column toColumn) throws SQLException
+    {
+        // sniff the value
+        Timestamp timestamp = from.getTimestamp(fromIndex);
+        if (timestamp != null) {
+            this.dateTime = timestamp;
+        }
+
+        super.getAndSet(from, fromIndex, toColumn);
+    }
+
+    @Override
+    public JsonNode encodeToJson()
+    {
+        String text = String.format(ENGLISH,
+                ISO_USEC_FORMAT,
+                dateTime.getYear() + 1900,
+                dateTime.getMonth() + 1,
+                dateTime.getDate(),
+                dateTime.getHours(),
+                dateTime.getMinutes(),
+                dateTime.getSeconds(),
+                dateTime.getNanos() / 1000);
+        return jsonNodeFactory.textNode(text);
+    }
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException
+    {
+        Matcher matcher = ISO_USEC_PATTERN.matcher(fromValue.asText());
+        if (!matcher.matches()) {
+            throw new ConfigException("Invalid timestamp without time zone pattern: " + fromValue);
+        }
+        Timestamp sqlDateTime = new Timestamp(
+                Integer.parseInt(matcher.group(1)) - 1900,  // year
+                Integer.parseInt(matcher.group(2)) - 1,  // month
+                Integer.parseInt(matcher.group(3)),  // day
+                Integer.parseInt(matcher.group(4)),  // hour
+                Integer.parseInt(matcher.group(5)),  // minute
+                Integer.parseInt(matcher.group(6)),  // second
+                Integer.parseInt(matcher.group(7)) * 1000);  // usec -> nsec
+        toStatement.setTimestamp(toIndex, sqlDateTime);
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-13 00:00:00.000000000 Z
+date: 2017-02-10 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -19,7 +19,7 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-input-jdbc-0.8.0.jar
+- classpath/embulk-input-jdbc-0.8.1.jar
 - lib/embulk/input/jdbc.rb
 - src/main/java/org/embulk/input/JdbcInputPlugin.java
 - src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
@@ -31,6 +31,7 @@ files:
 - src/main/java/org/embulk/input/jdbc/ToString.java
 - src/main/java/org/embulk/input/jdbc/ToStringMap.java
 - src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java
+- src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java
 - src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java
@@ -44,8 +45,8 @@ files:
 - src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
-- src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java
-- src/test/java/org/embulk/input/tester/EmbulkPluginTester.java
+- src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java
+- src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0

data/src/test/java/org/embulk/input/AbstractJdbcInputPluginTest.java

@@ -1,254 +0,0 @@
-package org.embulk.input;
-
-import static java.util.Locale.ENGLISH;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.net.URISyntaxException;
-import java.nio.charset.Charset;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.embulk.config.ConfigException;
-import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
-import org.embulk.input.tester.EmbulkPluginTester;
-import org.embulk.input.tester.EmbulkPluginTester.PluginDefinition;
-import org.yaml.snakeyaml.Yaml;
-
-import com.google.common.io.Files;
-
-public abstract class AbstractJdbcInputPluginTest
-{
-    private static final String CONFIG_FILE_NAME = "tests.yml";
-
-    protected boolean enabled;
-    // TODO:destroy EmbulkPluginTester after test
-    protected EmbulkPluginTester tester = new EmbulkPluginTester();
-    private String pluginName;
-    private Map<String, ?> testConfigurations;
-
-    protected AbstractJdbcInputPluginTest()
-    {
-        try {
-            prepare();
-        } catch (SQLException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    protected abstract void prepare() throws SQLException;
-
-
-    private Map<String, ?> getTestConfigs()
-    {
-        if (testConfigurations == null) {
-            for (PluginDefinition pluginDefinition : tester.getPlugins()) {
-                if (AbstractJdbcInputPlugin.class.isAssignableFrom(pluginDefinition.impl)) {
-                    pluginName = pluginDefinition.name;
-                    break;
-                }
-            }
-
-            Yaml yaml = new Yaml();
-            File configFile = new File(CONFIG_FILE_NAME);
-            if (!configFile.exists()) {
-                configFile = new File("../" + CONFIG_FILE_NAME);
-                if (!configFile.exists()) {
-                    throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't exist.",
-                            CONFIG_FILE_NAME));
-                }
-            }
-
-            try {
-                InputStreamReader reader = new InputStreamReader(new FileInputStream(configFile), Charset.forName("UTF8"));
-                try {
-                    Map<String, ?> allTestConfigs = (Map<String, ?>)yaml.load(reader);
-                    if (!allTestConfigs.containsKey(pluginName)) {
-                        throw new ConfigException(String.format(ENGLISH, "\"%s\" doesn't contain \"%s\" element.",
-                                CONFIG_FILE_NAME, pluginName));
-                    }
-                    testConfigurations = (Map<String, ?>)allTestConfigs.get(pluginName);
-                } finally {
-                    reader.close();
-                }
-            } catch (IOException e) {
-                throw new RuntimeException(e);
-            }
-        }
-        return testConfigurations;
-    }
-
-    protected Object getTestConfig(String name, boolean required)
-    {
-        Map<String, ?> testConfigs = getTestConfigs();
-        if (!testConfigs.containsKey(name)) {
-            if (required) {
-                throw new ConfigException(String.format(ENGLISH, "\"%s\" element in \"%s\" doesn't contain \"%s\" element.",
-                        pluginName, CONFIG_FILE_NAME, name));
-            }
-            return null;
-        }
-        return testConfigs.get(name);
-    }
-
-    protected Object getTestConfig(String name)
-    {
-        return getTestConfig(name, true);
-    }
-
-    protected String getHost()
-    {
-        return (String)getTestConfig("host");
-    }
-
-    protected int getPort()
-    {
-        return (Integer)getTestConfig("port");
-    }
-
-    protected String getUser()
-    {
-        return (String)getTestConfig("user");
-    }
-
-    protected String getPassword()
-    {
-        return (String)getTestConfig("password");
-    }
-
-    protected String getDatabase()
-    {
-        return (String)getTestConfig("database");
-    }
-
-    protected void dropTable(String table) throws SQLException
-    {
-        String sql = String.format("DROP TABLE %s", table);
-        executeSQL(sql, true);
-    }
-
-    protected List<List<Object>> select(String table) throws SQLException
-    {
-        try (Connection connection = connect()) {
-            try (Statement statement = connection.createStatement()) {
-                List<List<Object>> rows = new ArrayList<List<Object>>();
-                String sql = String.format("SELECT * FROM %s", table);
-                System.out.println(sql);
-                try (ResultSet resultSet = statement.executeQuery(sql)) {
-                    while (resultSet.next()) {
-                        List<Object> row = new ArrayList<Object>();
-                        for (int i = 1; i <= resultSet.getMetaData().getColumnCount(); i++) {
-                            row.add(getValue(resultSet, i));
-                        }
-                        rows.add(row);
-                    }
-                }
-                // cannot sort by CLOB, so sort by Java
-                Collections.sort(rows, new Comparator<List<Object>>() {
-                    @Override
-                    public int compare(List<Object> o1, List<Object> o2) {
-                        return o1.toString().compareTo(o2.toString());
-                    }
-                });
-                return rows;
-            }
-        }
-
-    }
-
-    protected Object getValue(ResultSet resultSet, int index) throws SQLException
-    {
-        return resultSet.getObject(index);
-    }
-
-    protected void executeSQL(String sql) throws SQLException
-    {
-        executeSQL(sql, false);
-    }
-
-    protected void executeSQL(String sql, boolean ignoreError) throws SQLException
-    {
-        if (!enabled) {
-            return;
-        }
-
-        try (Connection connection = connect()) {
-            try {
-                connection.setAutoCommit(true);
-
-                try (Statement statement = connection.createStatement()) {
-                    System.out.println(String.format("Execute SQL : \"%s\".", sql));
-                    statement.execute(sql);
-                }
-
-            } catch (SQLException e) {
-                if (!ignoreError) {
-                    throw e;
-                }
-            }
-        }
-    }
-
-    protected void test(String ymlPath) throws Exception
-    {
-        if (!enabled) {
-            return;
-        }
-
-        tester.run(convertYml(ymlPath));
-    }
-
-    protected String convertYml(String ymlName) throws Exception
-    {
-        StringBuilder builder = new StringBuilder();
-        Pattern pathPrefixPattern = Pattern.compile("^ *path(_prefix)?: '(.*)'$");
-        for (String line : Files.readLines(convertPath(ymlName), Charset.forName("UTF8"))) {
-            line = convertYmlLine(line);
-            Matcher matcher = pathPrefixPattern.matcher(line);
-            if (matcher.matches()) {
-                int group = 2;
-                builder.append(line.substring(0, matcher.start(group)));
-                builder.append(convertPath(matcher.group(group)).getAbsolutePath());
-                builder.append(line.substring(matcher.end(group)));
-            } else {
-                builder.append(line);
-            }
-            builder.append(System.lineSeparator());
-        }
-        return builder.toString();
-    }
-
-    protected String convertYmlLine(String line)
-    {
-        line = line.replaceAll("#host#", getHost());
-        line = line.replaceAll("#port#", Integer.toString(getPort()));
-        line = line.replaceAll("#database#", getDatabase());
-        line = line.replaceAll("#user#", getUser());
-        line = line.replaceAll("#password#", getPassword());
-        return line;
-    }
-
-    protected File convertPath(String name) throws URISyntaxException
-    {
-        return new File(getClass().getResource(name).toURI());
-    }
-
-    protected List<String> read(String path) throws IOException
-    {
-        return Files.readLines(new File(path), Charset.forName("UTF8"));
-    }
-
-    protected abstract Connection connect() throws SQLException;
-
-}

data/src/test/java/org/embulk/input/tester/EmbulkPluginTester.java

@@ -1,82 +0,0 @@
-package org.embulk.input.tester;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.embulk.EmbulkEmbed;
-import org.embulk.EmbulkEmbed.Bootstrap;
-import org.embulk.config.ConfigSource;
-import org.embulk.plugin.InjectedPluginSource;
-
-import com.google.inject.Binder;
-import com.google.inject.Module;
-
-public class EmbulkPluginTester
-{
-    public static class PluginDefinition
-    {
-        public final Class<?> iface;
-        public final String name;
-        public final Class<?> impl;
-
-
-        public PluginDefinition(Class<?> iface, String name, Class<?> impl)
-        {
-            this.iface = iface;
-            this.name = name;
-            this.impl = impl;
-        }
-
-    }
-
-    private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
-
-    private EmbulkEmbed embulk;
-
-    public EmbulkPluginTester()
-    {
-    }
-
-    public EmbulkPluginTester(Class<?> iface, String name, Class<?> impl)
-    {
-        addPlugin(iface, name, impl);
-    }
-
-    public void addPlugin(Class<?> iface, String name, Class<?> impl)
-    {
-        plugins.add(new PluginDefinition(iface, name, impl));
-    }
-
-    public List<PluginDefinition> getPlugins()
-    {
-        return plugins;
-    }
-
-    public void run(String yml) throws Exception
-    {
-        if (embulk == null) {
-            Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
-            bootstrap.addModules(new Module()
-            {
-                @Override
-                public void configure(Binder binder)
-                {
-                    for (PluginDefinition plugin : plugins) {
-                        InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
-                    }
-                }
-            });
-            embulk = bootstrap.initializeCloseable();
-        }
-        ConfigSource config = embulk.newConfigLoader().fromYamlString(yml);
-        embulk.run(config);
-    }
-
-    public void destroy() {
-        if (embulk != null) {
-            embulk.destroy();
-            embulk = null;
-        }
-    }
-
-}