embulk-input-jdbc 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9a672095825353519696e26638bf7b750406a720
-  data.tar.gz: f90651ce7041f7435cc07da9435dafe593b027cc
+  metadata.gz: bab76f99601161e312ae588c0f5d1c6ed7a5ca7c
+  data.tar.gz: 3394307ce36d292c46de24aafa14a79859cd6b13
 SHA512:
-  metadata.gz: 5dbd93a8b413d1ff2b3d53d61c04706a1696cc8a544805c094d9fdef05a8bd11072a98b39edf602b08af9ab679684220bd6699df2d0963ded92d514c7d6e7b2e
-  data.tar.gz: 9886411e1b6d7358669827291f53a4994b1a116123d724253c0f39ce9c5041413e8207b8b84ab02b4686c8775be6594f6e70a0e0d74a79190d013f06d0514b23
+  metadata.gz: ae4abde050d7ff724733096a16e0461a5d5b31d0877e0ab76ebe4b763c00507b9bca116ad19f7b27e9ce3870814b843589133f4aca0c2a530680594cc81261b4
+  data.tar.gz: a7b72823dd55164287445e38a2d3d5b2c18b76c638d1c8d8c38aa0d6cd5af987a55f818c43b8a3627c06653a3bbec6503855740a005fb97e6dafc647035cc2ca

data/src/main/java/org/embulk/input/JdbcInputPlugin.java

@@ -59,7 +59,7 @@ public class JdbcInputPlugin
             synchronized (loadedJarGlobs) {
                 String glob = t.getDriverPath().get();
                 if (!loadedJarGlobs.contains(glob)) {
-                    loadDriverJar(glob);
+                    addDriverJarToClasspath(glob);
                     loadedJarGlobs.add(glob);
                 }
             }

data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java

@@ -1,5 +1,6 @@
 package org.embulk.input.jdbc;
 
+import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
 import java.nio.file.Paths;
@@ -8,10 +9,12 @@ import java.sql.SQLException;
 
 import org.slf4j.Logger;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Optional;
 import com.google.common.base.Supplier;
 import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigException;
@@ -25,6 +28,7 @@ import org.embulk.config.TaskSource;
 import org.embulk.plugin.PluginClassLoader;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Column;
+import org.embulk.spi.DataException;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.InputPlugin;
 import org.embulk.spi.PageOutput;
@@ -33,8 +37,11 @@ import org.embulk.spi.Exec;
 import org.embulk.input.jdbc.getter.ColumnGetter;
 import org.embulk.input.jdbc.getter.ColumnGetterFactory;
 import org.embulk.input.jdbc.JdbcInputConnection.BatchSelect;
+import org.embulk.input.jdbc.JdbcInputConnection.PreparedQuery;
 import org.joda.time.DateTimeZone;
 
+import static java.util.Locale.ENGLISH;
+
 public abstract class AbstractJdbcInputPlugin
         implements InputPlugin
 {
@@ -49,6 +56,7 @@ public abstract class AbstractJdbcInputPlugin
         @Config("table")
         @ConfigDefault("null")
         public Optional<String> getTable();
+        public void setTable(Optional<String> normalizedTableName);
 
         @Config("query")
         @ConfigDefault("null")
@@ -66,10 +74,18 @@ public abstract class AbstractJdbcInputPlugin
         @ConfigDefault("null")
         public Optional<String> getOrderBy();
 
-        //// TODO See bellow.
-        //@Config("last_value")
-        //@ConfigDefault("null")
-        //public Optional<String> getLastValue();
+        @Config("incremental")
+        @ConfigDefault("false")
+        public boolean getIncremental();
+
+        @Config("incremental_columns")
+        @ConfigDefault("[]")
+        public List<String> getIncrementalColumns();
+        public void setIncrementalColumns(List<String> indexes);
+
+        @Config("last_record")
+        @ConfigDefault("null")
+        public Optional<List<JsonNode>> getLastRecord();
 
         // TODO limit_value is necessary to make sure repeated bulk load transactions
         //      don't a same record twice or miss records when the column
@@ -128,9 +144,15 @@ public abstract class AbstractJdbcInputPlugin
         @ConfigDefault("null")
         public Optional<String> getAfterSelect();
 
+        public PreparedQuery getBuiltQuery();
+        public void setBuiltQuery(PreparedQuery query);
+
         public JdbcSchema getQuerySchema();
         public void setQuerySchema(JdbcSchema schema);
 
+        public List<Integer> getIncrementalColumnIndexes();
+        public void setIncrementalColumnIndexes(List<Integer> indexes);
+
         @ConfigInject
         public BufferAllocator getBufferAllocator();
     }
@@ -149,12 +171,20 @@ public abstract class AbstractJdbcInputPlugin
     {
         PluginTask task = config.loadConfig(getTaskClass());
 
-        //if (task.getLastValue().isPresent() && !task.getOrderBy().isPresent()) {
-        //    throw new ConfigException("order_by parameter must be set if last_value parameter is set");
-        //}
+        if (task.getIncremental()) {
+            if (task.getOrderBy().isPresent()) {
+                throw new ConfigException("order_by option must not be set if incremental is true");
+            }
+        }
+        else {
+            if (!task.getIncrementalColumns().isEmpty()) {
+                throw new ConfigException("'incremental: true' must be set if incremental_columns is set");
+            }
+        }
 
         Schema schema;
         try (JdbcInputConnection con = newConnection(task)) {
+            // TODO incremental_columns is not set => get primary key
             schema = setupTask(con, task);
         } catch (SQLException ex) {
             throw Throwables.propagate(ex);
@@ -165,11 +195,60 @@ public abstract class AbstractJdbcInputPlugin
 
     private Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException
     {
+        if (task.getTable().isPresent()) {
+            String actualTableName = normalizeTableNameCase(con, task.getTable().get());
+            task.setTable(Optional.of(actualTableName));
+        }
+
         // build SELECT query and gets schema of its result
         String query = getQuery(task, con);
-        logger.info("SQL: " + query);
+
         JdbcSchema querySchema = con.getSchemaOfQuery(query);
         task.setQuerySchema(querySchema);
+        // query schema should not change after incremental query
+
+        PreparedQuery preparedQuery;
+        if (task.getIncremental()) {
+            // build incremental query
+
+            List<String> incrementalColumns = task.getIncrementalColumns();
+            if (incrementalColumns.isEmpty()) {
+                // incremental_columns is not set
+                if (!task.getTable().isPresent()) {
+                    throw new ConfigException("incremental_columns option must be set if incremental is true and custom query option is set");
+                }
+                // get primary keys from the target table to use them as incremental_columns
+                List<String> primaryKeys = con.getPrimaryKeys(task.getTable().get());
+                if (primaryKeys.isEmpty()) {
+                    throw new ConfigException(String.format(ENGLISH,
+                                "Primary key is not available at the table '%s'. incremental_columns option must be set",
+                                task.getTable().get()));
+                }
+                logger.info("Using primary keys as incremental_columns: {}", primaryKeys);
+                task.setIncrementalColumns(primaryKeys);
+                incrementalColumns = primaryKeys;
+            }
+
+            List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
+            task.setIncrementalColumnIndexes(incrementalColumnIndexes);
+
+            if (task.getLastRecord().isPresent()) {
+                List<JsonNode> lastRecord = task.getLastRecord().get();
+                if (lastRecord.size() != incrementalColumnIndexes.size()) {
+                    throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
+                }
+                preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, lastRecord);
+            }
+            else {
+                preparedQuery = con.buildIncrementalQuery(query, querySchema, incrementalColumnIndexes, null);
+            }
+        }
+        else {
+            task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
+            preparedQuery = new PreparedQuery(query, ImmutableList.<JdbcLiteral>of());
+        }
+
+        task.setBuiltQuery(preparedQuery);
 
         // validate column_options
         newColumnGetters(task, querySchema, null);
@@ -186,19 +265,63 @@ public abstract class AbstractJdbcInputPlugin
         return new Schema(columns.build());
     }
 
+    private String normalizeTableNameCase(JdbcInputConnection con, String tableName)
+        throws SQLException
+    {
+        if (con.tableExists(tableName)) {
+            return tableName;
+        } else {
+            String upperTableName = tableName.toUpperCase();
+            String lowerTableName = tableName.toLowerCase();
+            boolean upperExists = con.tableExists(upperTableName);
+            boolean lowerExists = con.tableExists(upperTableName);
+            if (upperExists && lowerExists) {
+                    throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
+                            tableName, upperTableName, lowerTableName));
+            } else if (upperExists) {
+                return upperTableName;
+            } else if (lowerExists) {
+                return lowerTableName;
+            } else {
+                // fallback to the given table name. this may throw error later at getSchemaOfQuery
+                return tableName;
+            }
+        }
+    }
+
+    private List<Integer> findIncrementalColumnIndexes(JdbcSchema schema, List<String> incrementalColumns)
+        throws SQLException
+    {
+        ImmutableList.Builder<Integer> builder = ImmutableList.builder();
+        for (String name : incrementalColumns) {
+            Optional<Integer> index = schema.findColumn(name);
+            if (index.isPresent()) {
+                builder.add(index.get());
+            }
+            else {
+                throw new ConfigException(String.format(ENGLISH,
+                        "Column name '%s' is in incremental_columns option does not exist",
+                        name));
+            }
+        }
+        return builder.build();
+    }
+
     private String getQuery(PluginTask task, JdbcInputConnection con) throws SQLException
     {
         if (task.getQuery().isPresent()) {
             if (task.getTable().isPresent() || task.getSelect().isPresent() ||
                     task.getWhere().isPresent() || task.getOrderBy().isPresent()) {
                 throw new ConfigException("'table', 'select', 'where' and 'order_by' parameters are unnecessary if 'query' parameter is set.");
+            } else if (!task.getIncrementalColumns().isEmpty() || task.getLastRecord().isPresent()) {
+                throw new ConfigException("'incremental_columns' and 'last_record' parameters are not supported if 'query' parameter is set.");
             }
             return task.getQuery().get();
         } else if (task.getTable().isPresent()) {
             return con.buildSelectQuery(task.getTable().get(), task.getSelect(),
                     task.getWhere(), task.getOrderBy());
         } else {
-            throw new ConfigException("'table' parameter is required (if 'query' parameter is not set)");
+            throw new ConfigException("'table' or 'query' parameter is required");
         }
     }
 
@@ -224,12 +347,11 @@ public abstract class AbstractJdbcInputPlugin
     protected ConfigDiff buildNextConfigDiff(PluginTask task, List<TaskReport> reports)
     {
         ConfigDiff next = Exec.newConfigDiff();
-        // TODO
-        //if (task.getOrderBy().isPresent()) {
-        //    // TODO when parallel execution is implemented, calculate the max last_value
-        //    //      from the all commit reports.
-        //    next.set("last_value", reports.get(0).get(JsonNode.class, "last_value"));
-        //}
+        if (reports.size() > 0 && reports.get(0).has("last_record")) {
+            next.set("last_record", reports.get(0).get(JsonNode.class, "last_record"));
+        } else if (task.getLastRecord().isPresent()) {
+            next.set("last_record", task.getLastRecord().get());
+        }
         return next;
     }
 
@@ -241,6 +363,42 @@ public abstract class AbstractJdbcInputPlugin
         // do nothing
     }
 
+    private static class LastRecordStore
+    {
+        private final List<Integer> columnIndexes;
+        private final JsonNode[] lastValues;
+        private final List<String> columnNames;
+
+        public LastRecordStore(List<Integer> columnIndexes, List<String> columnNames)
+        {
+            this.columnIndexes = columnIndexes;
+            this.lastValues = new JsonNode[columnIndexes.size()];
+            this.columnNames = columnNames;
+        }
+
+        public void accept(List<ColumnGetter> getters)
+            throws SQLException
+        {
+            for (int i = 0; i < columnIndexes.size(); i++) {
+                lastValues[i] = getters.get(columnIndexes.get(i)).encodeToJson();
+            }
+        }
+
+        public List<JsonNode> getList()
+        {
+            ImmutableList.Builder<JsonNode> builder = ImmutableList.builder();
+            for (int i = 0; i < lastValues.length; i++) {
+                if (lastValues[i] == null || lastValues[i].isNull()) {
+                    throw new DataException(String.format(ENGLISH,
+                            "incremental_columns can't include null values but the last row is null at column '%s'",
+                            columnNames.get(i)));
+                }
+                builder.add(lastValues[i]);
+            }
+            return builder.build();
+        }
+    }
+
     @Override
     public TaskReport run(TaskSource taskSource,
             Schema schema, int taskIndex,
@@ -248,22 +406,32 @@ public abstract class AbstractJdbcInputPlugin
     {
         PluginTask task = taskSource.loadTask(getTaskClass());
 
+        PreparedQuery builtQuery = task.getBuiltQuery();
         JdbcSchema querySchema = task.getQuerySchema();
         BufferAllocator allocator = task.getBufferAllocator();
         PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
 
+        long totalRows = 0;
+
+        LastRecordStore lastRecordStore = null;
+
         try (JdbcInputConnection con = newConnection(task)) {
-            try (BatchSelect cursor = con.newSelectCursor(getQuery(task, con), task.getFetchRows(), task.getSocketTimeout())) {
-                List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
+            List<ColumnGetter> getters = newColumnGetters(task, querySchema, pageBuilder);
+            try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
                 while (true) {
-                    // TODO run fetch() in another thread asynchronously
-                    // TODO retry fetch() if it failed (maybe order_by is required and unique_column(s) option is also required)
-                    boolean cont = fetch(cursor, getters, pageBuilder);
-                    if (!cont) {
+                    long rows = fetch(cursor, getters, pageBuilder);
+                    if (rows <= 0L) {
                         break;
                     }
+                    totalRows += rows;
                 }
             }
+
+            if (task.getIncremental() && totalRows > 0) {
+                lastRecordStore = new LastRecordStore(task.getIncrementalColumnIndexes(), task.getIncrementalColumns());
+                lastRecordStore.accept(getters);
+            }
+
             pageBuilder.finish();
 
             // after_select runs after pageBuilder.finish because pageBuilder.finish may fail.
@@ -284,10 +452,10 @@ public abstract class AbstractJdbcInputPlugin
         }
 
         TaskReport report = Exec.newTaskReport();
-        // TODO
-        //if (orderByColumn != null) {
-        //    report.set("last_value", lastValue);
-        //}
+        if (lastRecordStore != null) {
+            report.set("last_record", lastRecordStore.getList());
+        }
+
         return report;
     }
 
@@ -339,12 +507,12 @@ public abstract class AbstractJdbcInputPlugin
                     });
     }
 
-    private boolean fetch(BatchSelect cursor,
+    private long fetch(BatchSelect cursor,
             List<ColumnGetter> getters, PageBuilder pageBuilder) throws SQLException
     {
         ResultSet result = cursor.fetch();
         if (result == null || !result.next()) {
-            return false;
+            return 0;
         }
 
         List<Column> columns = pageBuilder.getSchema().getColumns();
@@ -362,7 +530,8 @@ public abstract class AbstractJdbcInputPlugin
                 reportRows *= 2;
             }
         } while (result.next());
-        return true;
+
+        return rows;
     }
 
     //// TODO move to embulk.spi.util?
@@ -402,10 +571,14 @@ public abstract class AbstractJdbcInputPlugin
     //    }
     //}
 
-    protected void loadDriverJar(String glob)
+    protected void addDriverJarToClasspath(String glob)
     {
         // TODO match glob
         PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
+        Path path = Paths.get(glob);
+        if (!path.toFile().exists()) {
+             throw new ConfigException("The specified driver jar doesn't exist: " + glob);
+        }
         loader.addPath(Paths.get(glob));
     }
 }

data/src/main/java/org/embulk/input/jdbc/JdbcColumn.java

@@ -44,7 +44,6 @@ public class JdbcColumn
         return sqlType;
     }
 
-
     @JsonProperty("precision")
     public int getPrecision()
     {

data/src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java

@@ -11,12 +11,19 @@ import java.util.Set;
 
 import org.embulk.config.ConfigException;
 import org.embulk.spi.Exec;
+import org.embulk.input.jdbc.getter.ColumnGetter;
 import org.slf4j.Logger;
 
+import java.util.List;
+import java.util.ArrayList;
+import static java.util.Locale.ENGLISH;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSet.Builder;
 
 public class JdbcInputConnection
         implements AutoCloseable
@@ -57,6 +64,20 @@ public class JdbcInputConnection
         }
     }
 
+    public List<String> getPrimaryKeys(String tableName) throws SQLException
+    {
+        ResultSet rs = databaseMetaData.getPrimaryKeys(null, schemaName, tableName);
+        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        try {
+            while(rs.next()) {
+                builder.add(rs.getString("COLUMN_NAME"));
+            }
+        } finally {
+            rs.close();
+        }
+        return builder.build();
+    }
+
     protected JdbcSchema getSchemaOfResultMetadata(ResultSetMetaData metadata) throws SQLException
     {
         ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
@@ -72,19 +93,70 @@ public class JdbcInputConnection
         return new JdbcSchema(columns.build());
     }
 
-    public BatchSelect newSelectCursor(String query, int fetchRows, int queryTimeout) throws SQLException
+    public static class PreparedQuery
     {
-        return newBatchSelect(query, fetchRows, queryTimeout);
+        private final String query;
+        private final List<JdbcLiteral> parameters;
+
+        @JsonCreator
+        public PreparedQuery(
+                @JsonProperty("query") String query,
+                @JsonProperty("parameters") List<JdbcLiteral> parameters)
+        {
+            this.query = query;
+            this.parameters = parameters;
+        }
+
+        @JsonProperty("query")
+        public String getQuery()
+        {
+            return query;
+        }
+
+        @JsonProperty("parameters")
+        public List<JdbcLiteral> getParameters()
+        {
+            return parameters;
+        }
     }
 
-    protected BatchSelect newBatchSelect(String query, int fetchRows, int queryTimeout) throws SQLException
+    public BatchSelect newSelectCursor(PreparedQuery preparedQuery,
+            List<ColumnGetter> getters,
+            int fetchRows, int queryTimeout) throws SQLException
     {
+        return newBatchSelect(preparedQuery, getters, fetchRows, queryTimeout);
+    }
+
+    protected BatchSelect newBatchSelect(PreparedQuery preparedQuery,
+            List<ColumnGetter> getters,
+            int fetchRows, int queryTimeout) throws SQLException
+    {
+        String query = preparedQuery.getQuery();
+        List<JdbcLiteral> params = preparedQuery.getParameters();
+
         PreparedStatement stmt = connection.prepareStatement(query);
         stmt.setFetchSize(fetchRows);
         stmt.setQueryTimeout(queryTimeout);
+        logger.info("SQL: " + query);
+        if (!params.isEmpty()) {
+            logger.info("Parameters: {}", params);
+            prepareParameters(stmt, getters, params);
+        }
         return new SingleSelect(stmt);
     }
 
+    protected void prepareParameters(PreparedStatement stmt, List<ColumnGetter> getters,
+            List<JdbcLiteral> parameters)
+        throws SQLException
+    {
+        for (int i = 0; i < parameters.size(); i++) {
+            JdbcLiteral literal = parameters.get(i);
+            ColumnGetter getter = getters.get(literal.getColumnIndex());
+            int index = i + 1;  // JDBC column index begins from 1
+            getter.decodeFromJsonTo(stmt, index, literal.getValue());
+        }
+    }
+
     public interface BatchSelect
             extends AutoCloseable
     {
@@ -156,72 +228,87 @@ public class JdbcInputConnection
     }
 
     public String buildSelectQuery(String tableName,
-            Optional<String> selectColumnList, Optional<String> whereCondition,
-            Optional<String> orderByColumn) throws SQLException
-    {
-        String actualTableName;
-        if (tableExists(tableName)) {
-            actualTableName = tableName;
-        } else {
-            String upperTableName = tableName.toUpperCase();
-            String lowerTableName = tableName.toLowerCase();
-            if (tableExists(upperTableName)) {
-                if (tableExists(lowerTableName)) {
-                    throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
-                            tableName, upperTableName, lowerTableName));
-                } else {
-                    actualTableName = upperTableName;
-                }
-            } else {
-                if (tableExists(lowerTableName)) {
-                    actualTableName = lowerTableName;
-                } else {
-                    actualTableName = tableName;
-                }
-            }
-        }
-
+            Optional<String> selectExpression, Optional<String> whereCondition,
+            Optional<String> orderByExpression) throws SQLException
+    {
         StringBuilder sb = new StringBuilder();
 
         sb.append("SELECT ");
-        sb.append(selectColumnList.or("*"));
-        sb.append(" FROM ").append(buildTableName(actualTableName));
+        sb.append(selectExpression.or("*"));
+        sb.append(" FROM ").append(buildTableName(tableName));
 
         if (whereCondition.isPresent()) {
             sb.append(" WHERE ").append(whereCondition.get());
         }
 
-        if (orderByColumn.isPresent()) {
-            String actualOrderByColumn;
-            Set<String> columnNames = getColumnNames(actualTableName);
-            if (columnNames.contains(orderByColumn.get())) {
-                actualOrderByColumn = orderByColumn.get();
-            } else {
-                String upperOrderByColumn = orderByColumn.get().toUpperCase();
-                String lowerOrderByColumn = orderByColumn.get().toLowerCase();
-                if (columnNames.contains(upperOrderByColumn)) {
-                    if (columnNames.contains(lowerOrderByColumn)) {
-                        throw new ConfigException(String.format("Cannot specify order-by colum '%s' because both '%s' and '%s' exist.",
-                                orderByColumn.get(), upperOrderByColumn, lowerOrderByColumn));
-                    } else {
-                        actualOrderByColumn = upperOrderByColumn;
-                    }
-                } else {
-                    if (columnNames.contains(lowerOrderByColumn)) {
-                        actualOrderByColumn = lowerOrderByColumn;
-                    } else {
-                        actualOrderByColumn = orderByColumn.get();
-                    }
+        if (orderByExpression.isPresent()) {
+            sb.append(" ORDER BY ").append(orderByExpression.get());
+        }
+
+        return sb.toString();
+    }
+
+    public PreparedQuery buildIncrementalQuery(String rawQuery, JdbcSchema querySchema,
+            List<Integer> incrementalColumnIndexes, List<JsonNode> incrementalValues) throws SQLException
+    {
+        StringBuilder sb = new StringBuilder();
+        ImmutableList.Builder<JdbcLiteral> parameters = ImmutableList.builder();
+
+        sb.append("SELECT * FROM (");
+        sb.append(truncateStatementDelimiter(rawQuery));
+        sb.append(") embulk_incremental_");
+        if (incrementalValues != null) {
+            sb.append(" WHERE ");
+
+            List<String> leftColumnNames = new ArrayList<>();
+            List<JdbcLiteral> rightLiterals = new ArrayList<>();
+            for (int n = 0; n < incrementalColumnIndexes.size(); n++) {
+                int columnIndex = incrementalColumnIndexes.get(n);
+                JsonNode value = incrementalValues.get(n);
+                leftColumnNames.add(querySchema.getColumnName(columnIndex));
+                rightLiterals.add(new JdbcLiteral(columnIndex, value));
+            }
+
+            for (int n = 0; n < leftColumnNames.size(); n++) {
+                if (n > 0) {
+                    sb.append(" OR ");
                 }
+                sb.append("(");
+
+                for (int i = 0; i < n; i++) {
+                    sb.append(quoteIdentifierString(leftColumnNames.get(i)));
+                    sb.append(" = ?");
+                    parameters.add(rightLiterals.get(i));
+                    sb.append(" AND ");
+                }
+                sb.append(quoteIdentifierString(leftColumnNames.get(n)));
+                sb.append(" > ?");
+                parameters.add(rightLiterals.get(n));
+
+                sb.append(")");
             }
+        }
+        sb.append(" ORDER BY ");
 
-            sb.append("ORDER BY ").append(quoteIdentifierString(actualOrderByColumn)).append(" ASC");
+        boolean first = true;
+        for (int i : incrementalColumnIndexes) {
+            if (first) {
+                first = false;
+            } else {
+                sb.append(", ");
+            }
+            sb.append(quoteIdentifierString(querySchema.getColumnName(i)));
         }
 
-        return sb.toString();
+        return new PreparedQuery(sb.toString(), parameters.build());
+    }
+
+    protected String truncateStatementDelimiter(String rawQuery) throws SQLException
+    {
+        return rawQuery.replaceAll(";\\s*$", "");
     }
 
-    private boolean tableExists(String tableName) throws SQLException
+    public boolean tableExists(String tableName) throws SQLException
     {
         try (ResultSet rs = connection.getMetaData().getTables(null, schemaName, tableName, null)) {
             return rs.next();
@@ -230,7 +317,7 @@ public class JdbcInputConnection
 
     private Set<String> getColumnNames(String tableName) throws SQLException
     {
-        Builder<String> columnNamesBuilder = ImmutableSet.builder();
+        ImmutableSet.Builder<String> columnNamesBuilder = ImmutableSet.builder();
         try (ResultSet rs = connection.getMetaData().getColumns(null, schemaName, tableName, null)) {
             while (rs.next()) {
                 columnNamesBuilder.add(rs.getString("COLUMN_NAME"));

data/src/main/java/org/embulk/input/jdbc/JdbcLiteral.java

@@ -0,0 +1,38 @@
+package org.embulk.input.jdbc;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+
+public class JdbcLiteral
+{
+    private final int columnIndex;
+    private final JsonNode value;
+
+    @JsonCreator
+    public JdbcLiteral(
+            @JsonProperty("columnIndex") int columnIndex,
+            @JsonProperty("value") JsonNode value)
+    {
+        this.columnIndex = columnIndex;
+        this.value = value;
+    }
+
+    @JsonProperty("columnIndex")
+    public int getColumnIndex()
+    {
+        return columnIndex;
+    }
+
+    @JsonProperty("value")
+    public JsonNode getValue()
+    {
+        return value;
+    }
+
+    @Override
+    public String toString()
+    {
+        return value.toString();
+    }
+}

data/src/main/java/org/embulk/input/jdbc/JdbcSchema.java

@@ -1,6 +1,7 @@
 package org.embulk.input.jdbc;
 
 import java.util.List;
+import com.google.common.base.Optional;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonValue;
 
@@ -34,4 +35,21 @@ public class JdbcSchema
     {
         return columns.get(i).getName();
     }
+
+    public Optional<Integer> findColumn(String caseInsensitiveName)
+    {
+        // find by case sensitive first
+        for (int i = 0; i < columns.size(); i++) {
+            if (getColumn(i).getName().equals(caseInsensitiveName)) {
+                return Optional.of(i);
+            }
+        }
+        // find by case insensitive
+        for (int i = 0; i < columns.size(); i++) {
+            if (getColumn(i).getName().equalsIgnoreCase(caseInsensitiveName)) {
+                return Optional.of(i);
+            }
+        }
+        return Optional.absent();
+    }
 }

data/src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java

@@ -1,14 +1,21 @@
 package org.embulk.input.jdbc.getter;
 
 import java.sql.ResultSet;
+import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import org.embulk.spi.Column;
 import org.embulk.spi.ColumnVisitor;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.type.Type;
+import org.embulk.spi.DataException;
+import static java.util.Locale.ENGLISH;
 
 public abstract class AbstractColumnGetter implements ColumnGetter, ColumnVisitor
 {
+    protected static final JsonNodeFactory jsonNodeFactory = JsonNodeFactory.instance;
+
     protected final PageBuilder to;
     private final Type toType;
 
@@ -20,8 +27,8 @@ public abstract class AbstractColumnGetter implements ColumnGetter, ColumnVisito
 
     @Override
     public void getAndSet(ResultSet from, int fromIndex,
-            Column toColumn) throws SQLException {
-
+            Column toColumn) throws SQLException
+    {
         fetch(from, fromIndex);
         if (from.wasNull()) {
             to.setNull(toColumn);
@@ -79,4 +86,20 @@ public abstract class AbstractColumnGetter implements ColumnGetter, ColumnVisito
 
     protected abstract Type getDefaultToType();
 
+    @Override
+    public JsonNode encodeToJson()
+    {
+        throw new DataException(String.format(ENGLISH,
+                            "Column type '%s' set at incremental_columns option is not supported",
+                            getToType()));
+    }
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException
+    {
+        throw new DataException(String.format(ENGLISH,
+                            "Converting last_record value %s to column index %d is not supported",
+                            fromValue.toString(), toIndex));
+    }
 }

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java

@@ -2,6 +2,8 @@ package org.embulk.input.jdbc.getter;
 
 import java.sql.ResultSet;
 import java.sql.SQLException;
+import java.sql.PreparedStatement;
+import com.fasterxml.jackson.databind.JsonNode;
 import org.embulk.spi.Column;
 import org.embulk.spi.type.Type;
 
@@ -11,4 +13,9 @@ public interface ColumnGetter
             Column toColumn) throws SQLException;
 
     public Type getToType();
+
+    public JsonNode encodeToJson();
+
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException;
 }

data/src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java

@@ -14,6 +14,8 @@ import org.embulk.spi.type.TimestampType;
 import org.embulk.spi.type.Type;
 import org.joda.time.DateTimeZone;
 
+import static java.util.Locale.ENGLISH;
+
 public class ColumnGetterFactory
 {
     protected final PageBuilder to;
@@ -58,7 +60,8 @@ public class ColumnGetterFactory
         case "decimal":
             return new BigDecimalColumnGetter(to, toType);
         default:
-            throw new ConfigException(String.format("Unknown value_type '%s' for column '%s'", option.getValueType(), column.getName()));
+            throw new ConfigException(String.format(ENGLISH,
+                        "Unknown value_type '%s' for column '%s'", option.getValueType(), column.getName()));
         }
     }
 
@@ -185,7 +188,8 @@ public class ColumnGetterFactory
     private static UnsupportedOperationException unsupportedOperationException(JdbcColumn column)
     {
         throw new UnsupportedOperationException(
-                String.format("Unsupported type %s (sqlType=%d) of '%s' column. Please exclude the column from 'select:' option.",
-                    column.getTypeName(), column.getSqlType(), column.getName()));
+                String.format(ENGLISH,
+                    "Unsupported type %s (sqlType=%d) of '%s' column. Please add '%s: {type: string}' to 'column_options: {...}' option to convert the values to strings, or exclude the column from 'select:' option",
+                    column.getTypeName(), column.getSqlType(), column.getName(), column.getName()));
     }
 }

data/src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java

@@ -1,7 +1,9 @@
 package org.embulk.input.jdbc.getter;
 
 import java.sql.ResultSet;
+import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import com.fasterxml.jackson.databind.JsonNode;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.type.Type;
@@ -53,4 +55,16 @@ public class LongColumnGetter
         to.setString(column, Long.toString(value));
     }
 
+    @Override
+    public JsonNode encodeToJson()
+    {
+        return jsonNodeFactory.numberNode(value);
+    }
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException
+    {
+        toStatement.setLong(toIndex, fromValue.asLong());
+    }
 }

data/src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java

@@ -1,7 +1,9 @@
 package org.embulk.input.jdbc.getter;
 
 import java.sql.ResultSet;
+import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import com.fasterxml.jackson.databind.JsonNode;
 import org.embulk.spi.Column;
 import org.embulk.spi.PageBuilder;
 import org.embulk.spi.json.JsonParseException;
@@ -79,4 +81,16 @@ public class StringColumnGetter
         to.setString(column, value);
     }
 
+    @Override
+    public JsonNode encodeToJson()
+    {
+        return jsonNodeFactory.textNode(value);
+    }
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+        throws SQLException
+    {
+        toStatement.setString(toIndex, fromValue.asText());
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-jdbc
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-08-26 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -18,12 +18,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - build.gradle
+- classpath/embulk-input-jdbc-0.7.3.jar
 - lib/embulk/input/jdbc.rb
 - src/main/java/org/embulk/input/JdbcInputPlugin.java
 - src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
 - src/main/java/org/embulk/input/jdbc/JdbcColumn.java
 - src/main/java/org/embulk/input/jdbc/JdbcColumnOption.java
 - src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java
+- src/main/java/org/embulk/input/jdbc/JdbcLiteral.java
 - src/main/java/org/embulk/input/jdbc/JdbcSchema.java
 - src/main/java/org/embulk/input/jdbc/ToString.java
 - src/main/java/org/embulk/input/jdbc/ToStringMap.java
@@ -42,7 +44,6 @@ files:
 - src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
 - src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
 - src/test/java/org/embulk/input/EmbulkPluginTester.java
-- classpath/embulk-input-jdbc-0.7.2.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:
 - Apache 2.0
@@ -53,17 +54,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: JDBC input plugin for Embulk