embulk-input-athena 0.1.0 → 0.1.6

data/src/main/resources/log4j.properties

@@ -0,0 +1,4 @@
+log4j.rootLogger=INFO, ROOT
+
+log4j.appender.ROOT=org.apache.log4j.varia.NullAppender
+log4j.appender.ROOT.layout=org.apache.log4j.PatternLayout

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-athena
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.6
 platform: ruby
 authors:
 - shinji19
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-16 00:00:00.000000000 Z
+date: 2021-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Loads records from Athena.
@@ -45,49 +45,12 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Dockerfile
 - LICENSE
 - README.md
 - build.gradle
-- config/checkstyle/checkstyle.xml
-- config/checkstyle/default.xml
-- docker-compose.yml
-- gradle/wrapper/gradle-wrapper.jar
-- gradle/wrapper/gradle-wrapper.properties
-- gradlew
-- gradlew.bat
-- lib/embulk/input/athena.rb
-- src/main/java/org/embulk/input/athena/AthenaInputConnection.java
-- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
-- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
-- src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
-- src/main/java/org/embulk/input/jdbc/JdbcColumn.java
-- src/main/java/org/embulk/input/jdbc/JdbcColumnOption.java
-- src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java
-- src/main/java/org/embulk/input/jdbc/JdbcLiteral.java
-- src/main/java/org/embulk/input/jdbc/JdbcSchema.java
-- src/main/java/org/embulk/input/jdbc/Ssl.java
-- src/main/java/org/embulk/input/jdbc/ToString.java
-- src/main/java/org/embulk/input/jdbc/ToStringMap.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java
-- src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
-- src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
+- classpath/AthenaJDBC41-1.1.0.jar
 - classpath/aws-java-sdk-1.11.301.jar
 - classpath/aws-java-sdk-acm-1.11.301.jar
 - classpath/aws-java-sdk-alexaforbusiness-1.11.301.jar
@@ -217,7 +180,8 @@ files:
 - classpath/aws-java-sdk-xray-1.11.301.jar
 - classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-athena-0.1.0.jar
+- classpath/embulk-input-athena-0.1.6.jar
+- classpath/embulk-input-jdbc-0.9.1.jar
 - classpath/httpclient-4.5.5.jar
 - classpath/httpcore-4.4.9.jar
 - classpath/ion-java-1.0.2.jar
@@ -231,6 +195,18 @@ files:
 - classpath/netty-handler-4.1.17.Final.jar
 - classpath/netty-resolver-4.1.17.Final.jar
 - classpath/netty-transport-4.1.17.Final.jar
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- docker-compose.yml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/athena.rb
+- src/main/java/org/embulk/input/athena/AthenaInputConnection.java
+- src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
+- src/main/resources/log4j.properties
+- src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
 homepage: https://github.com/shinji19/embulk-input-athena
 licenses:
 - MIT
@@ -241,17 +217,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Athena input plugin for Embulk

data/src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1

@@ -1,192 +0,0 @@
-package org.embulk.input.athena;
-
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
-import java.util.Properties;
-
-import com.google.common.base.Optional;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
-import org.embulk.input.jdbc.JdbcInputConnection;
-import org.embulk.spi.Exec;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.SchemaConfig;
-
-public class AthenaInputPlugin
-        extends AbstractJdbcInputPlugin
-{
-    public interface AthenaPluginTask
-            extends AbstractJdbcInputPlugin.PluginTask
-    {
-        @Config("driver_path")
-        @ConfigDefault("null")
-        public Optional<String> getDriverPath();
-
-        // athena_url (required string)
-        @Config("athena_url")
-        public String getAthenaUrl();
-
-        // s3_staging_dir (required string)
-        @Config("s3_staging_dir")
-        public String getS3StagingDir();
-
-        // access_key (required string)
-        @Config("access_key")
-        public String getAccessKey();        
-
-        // secret_key (required string)
-        @Config("secret_key")
-        public String getSecretKey();
-
-        // configuration option 2 (optional string, null is not allowed)
-        // @Config("option2")
-        // @ConfigDefault("\"myvalue\"")
-        // public String getOption2();
-
-        // configuration option 3 (optional string, null is allowed)
-        // @Config("option3")
-        // @ConfigDefault("null")
-        // public Optional<String> getOption3();
-
-        // if you get schema from config
-        // @Config("columns")
-        // public SchemaConfig getColumns();
-    }
-
-    @Override
-    protected Class<? extends PluginTask> getTaskClass() {
-        return AthenaPluginTask.class;
-    }
-
-    @Override
-    protected AthenaInputConnection newConnection(PluginTask pluginTask) throws SQLException {
-        AthenaPluginTask task = (AthenaPluginTask) pluginTask;
-        loadDriver("com.amazonaws.athena.jdbc.AthenaDriver", task.getDriverPath());
-        //Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
-        Properties properties = new Properties();
-        properties.put("s3_staging_dir", task.getS3StagingDir());
-        properties.put("user", task.getAccessKey());
-        properties.put("password", task.getSecretKey());
-        properties.putAll(task.getOptions());
-
-        Connection connection = DriverManager.getConnection(task.getAthenaUrl(), properties);
-        try {
-            AthenaInputConnection c = new AthenaInputConnection(connection);
-            connection = null;
-            return c;
-        } finally {
-            if (connection != null) {
-                connection.close();
-            }
-        }
-    }
-
-/*
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            InputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(PluginTask.class);
-
-        // Schema schema = task.getColumns().toSchema();
-        Schema schema = Schema.builder().build();
-        int taskCount = 1;  // number of run() method calls
-
-        return resume(task.dump(), schema, taskCount, control);
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control)
-    {
-        control.run(taskSource, schema, taskCount);
-        return Exec.newConfigDiff();
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    {
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource,
-            Schema schema, int taskIndex,
-            PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
-        
-        // Write your code here :)
-
-        Connection connection = null;
-        Statement statement = null;
-        try {
-            connection = getAthenaConnection(task);
-            statement = connection.createStatement();
-            ResultSet resultSet = statement.executeQuery("select * from default.sample");
-
-            while(resultSet.next()){
-                String statusCode = resultSet.getString("created_at");
-                System.out.println("st code" + statusCode);
-            }
-            resultSet.close();
-            connection.close();
-        } catch (Exception e){
-            e.printStackTrace();
-        } finally {
-            try {
-                if (statement != null)
-                    statement.close();
-            } catch (Exception ex) {
-   
-            }
-            try {
-                if (connection != null)
-                    connection.close();
-            } catch (Exception ex) {
-                ex.printStackTrace();
-            }
-        }
-
-        return Exec.newTaskReport();
-    }
-
-    @Override
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return Exec.newConfigDiff();
-    }
-*/
-/*
-    protected Connection getAthenaConnection(PluginTask task) throws ClassNotFoundException, SQLException{
-        Class.forName("com.amazonaws.athena.jdbc.AthenaDriver");
-        Properties properties = new Properties();
-        properties.put("s3_staging_dir", task.getS3StagingDir());
-        properties.put("user", task.getAccessKey());
-        properties.put("password", task.getSecretKey());
-        if (task.getLogPath() != null){
-            properties.put("log_path", task.getLogPath());
-        }
-        if (task.getLogLevel() != null){
-            properties.put("log_level", task.getLogLevel());
-        }
-
-        return DriverManager.getConnection(task.getAthenaUrl(), properties);
-    }
-    */
-}

data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java

@@ -1,674 +0,0 @@
-package org.embulk.input.jdbc;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.nio.file.Paths;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import org.slf4j.Logger;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.plugin.PluginClassLoader;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Exec;
-import org.embulk.input.jdbc.getter.ColumnGetter;
-import org.embulk.input.jdbc.getter.ColumnGetterFactory;
-import org.embulk.input.jdbc.JdbcInputConnection.BatchSelect;
-import org.embulk.input.jdbc.JdbcInputConnection.PreparedQuery;
-import org.joda.time.DateTimeZone;
-
-import static java.util.Locale.ENGLISH;
-
-public abstract class AbstractJdbcInputPlugin
-        implements InputPlugin
-{
-    protected final Logger logger = Exec.getLogger(getClass());
-
-    public interface PluginTask extends Task
-    {
-        @Config("options")
-        @ConfigDefault("{}")
-        public ToStringMap getOptions();
-
-        @Config("table")
-        @ConfigDefault("null")
-        public Optional<String> getTable();
-        public void setTable(Optional<String> normalizedTableName);
-
-        @Config("query")
-        @ConfigDefault("null")
-        public Optional<String> getQuery();
-
-        @Config("select")
-        @ConfigDefault("null")
-        public Optional<String> getSelect();
-
-        @Config("where")
-        @ConfigDefault("null")
-        public Optional<String> getWhere();
-
-        @Config("order_by")
-        @ConfigDefault("null")
-        public Optional<String> getOrderBy();
-
-        @Config("incremental")
-        @ConfigDefault("false")
-        public boolean getIncremental();
-
-        @Config("incremental_columns")
-        @ConfigDefault("[]")
-        public List<String> getIncrementalColumns();
-        public void setIncrementalColumns(List<String> indexes);
-
-        @Config("last_record")
-        @ConfigDefault("null")
-        public Optional<List<JsonNode>> getLastRecord();
-
-        // TODO limit_value is necessary to make sure repeated bulk load transactions
-        //      don't a same record twice or miss records when the column
-        //      specified at order_by parameter is not unique.
-        //      For example, if the order_by column is "timestamp created_at"
-        //      column whose precision is second, the table can include multiple
-        //      records with the same created_at time. At the first bulk load
-        //      transaction, it loads a record with created_at=2015-01-02 00:00:02.
-        //      Then next transaction will use WHERE created_at > '2015-01-02 00:00:02'.
-        //      However, if another record with created_at=2014-01-01 23:59:59 is
-        //      inserted between the 2 transactions, the new record will be skipped.
-        //      To prevent this scenario, we want to specify
-        //      limit_value=2015-01-02 00:00:00 (exclusive). With this way, as long as
-        //      a transaction runs after 2015-01-02 00:00:00 + some minutes, we don't
-        //      skip records. Ideally, to automate the scheduling, we want to set
-        //      limit_value="today".
-        //
-        //@Config("limit_value")
-        //@ConfigDefault("null")
-        //public Optional<String> getLimitValue();
-
-        //// TODO probably limit_rows is unnecessary as long as this has
-        //        supports parallel execution (partition_by option) and resuming.
-        //@Config("limit_rows")
-        //@ConfigDefault("null")
-        //public Optional<Integer> getLimitRows();
-
-        @Config("connect_timeout")
-        @ConfigDefault("300")
-        public int getConnectTimeout();
-
-        @Config("socket_timeout")
-        @ConfigDefault("1800")
-        public int getSocketTimeout();
-
-        @Config("fetch_rows")
-        @ConfigDefault("10000")
-        // TODO set minimum number
-        public int getFetchRows();
-
-        // TODO parallel execution using "partition_by" config
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        public Map<String, JdbcColumnOption> getColumnOptions();
-
-        @Config("default_timezone")
-        @ConfigDefault("\"UTC\"")
-        public DateTimeZone getDefaultTimeZone();
-
-        @Config("default_column_options")
-        @ConfigDefault("{}")
-        public Map<String, JdbcColumnOption> getDefaultColumnOptions();
-
-        @Config("after_select")
-        @ConfigDefault("null")
-        public Optional<String> getAfterSelect();
-
-        public PreparedQuery getBuiltQuery();
-        public void setBuiltQuery(PreparedQuery query);
-
-        public JdbcSchema getQuerySchema();
-        public void setQuerySchema(JdbcSchema schema);
-
-        public List<Integer> getIncrementalColumnIndexes();
-        public void setIncrementalColumnIndexes(List<Integer> indexes);
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    // for subclasses to add @Config
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return PluginTask.class;
-    }
-
-    protected abstract JdbcInputConnection newConnection(PluginTask task) throws SQLException;
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            InputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(getTaskClass());
-
-        if (task.getIncremental()) {
-            if (task.getOrderBy().isPresent()) {
-                throw new ConfigException("order_by option must not be set if incremental is true");
-            }
-        }
-        else {
-            if (!task.getIncrementalColumns().isEmpty()) {
-                throw new ConfigException("'incremental: true' must be set if incremental_columns is set");
-            }
-        }
-
-        Schema schema;
-        try (JdbcInputConnection con = newConnection(task)) {
-            con.showDriverVersion();
-
-            // TODO incremental_columns is not set => get primary key
-            schema = setupTask(con, task);
-        } catch (SQLException ex) {
-            throw Throwables.propagate(ex);
-        }
-
-        return buildNextConfigDiff(task, control.run(task.dump(), schema, 1));
-    }
-
-    protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException
-    {
-        if (task.getTable().isPresent()) {
-            String actualTableName = normalizeTableNameCase(con, task.getTable().get());
-            task.setTable(Optional.of(actualTableName));
-        }
-
-        // build SELECT query and gets schema of its result
-        String rawQuery = getRawQuery(task, con);
-
-        JdbcSchema querySchema = con.getSchemaOfQuery(rawQuery);
-        task.setQuerySchema(querySchema);
-        // query schema should not change after incremental query
-
-        PreparedQuery preparedQuery;
-        if (task.getIncremental()) {
-            // build incremental query
-
-            List<String> incrementalColumns = task.getIncrementalColumns();
-            if (incrementalColumns.isEmpty()) {
-                // incremental_columns is not set
-                if (!task.getTable().isPresent()) {
-                    throw new ConfigException("incremental_columns option must be set if incremental is true and custom query option is set");
-                }
-                // get primary keys from the target table to use them as incremental_columns
-                List<String> primaryKeys = con.getPrimaryKeys(task.getTable().get());
-                if (primaryKeys.isEmpty()) {
-                    throw new ConfigException(String.format(ENGLISH,
-                                "Primary key is not available at the table '%s'. incremental_columns option must be set",
-                                task.getTable().get()));
-                }
-                logger.info("Using primary keys as incremental_columns: {}", primaryKeys);
-                task.setIncrementalColumns(primaryKeys);
-                incrementalColumns = primaryKeys;
-            }
-
-            List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
-            task.setIncrementalColumnIndexes(incrementalColumnIndexes);
-
-            List<JsonNode> lastRecord;
-            if (task.getLastRecord().isPresent()) {
-                lastRecord = task.getLastRecord().get();
-                if (lastRecord.size() != incrementalColumnIndexes.size()) {
-                    throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
-                }
-            }
-            else {
-                lastRecord = null;
-            }
-
-            if (task.getQuery().isPresent()) {
-                preparedQuery = con.wrapIncrementalQuery(rawQuery, querySchema, incrementalColumnIndexes, lastRecord);
-            }
-            else {
-                preparedQuery = con.rebuildIncrementalQuery(
-                        task.getTable().get(), task.getSelect(),
-                        task.getWhere(),
-                        querySchema, incrementalColumnIndexes, lastRecord);
-            }
-        }
-        else {
-            task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
-            preparedQuery = new PreparedQuery(rawQuery, ImmutableList.<JdbcLiteral>of());
-        }
-
-        task.setBuiltQuery(preparedQuery);
-
-        // validate column_options
-        newColumnGetters(con, task, querySchema, null);
-
-        ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
-        ImmutableList.Builder<Column> columns = ImmutableList.builder();
-        for (int i = 0; i < querySchema.getCount(); i++) {
-            JdbcColumn column = querySchema.getColumn(i);
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
-            columns.add(new Column(i,
-                    column.getName(),
-                    factory.newColumnGetter(con, task, column, columnOption).getToType()));
-        }
-        return new Schema(columns.build());
-    }
-
-    private String normalizeTableNameCase(JdbcInputConnection con, String tableName)
-        throws SQLException
-    {
-        if (con.tableExists(tableName)) {
-            return tableName;
-        } else {
-            String upperTableName = tableName.toUpperCase();
-            String lowerTableName = tableName.toLowerCase();
-            boolean upperExists = con.tableExists(upperTableName);
-            boolean lowerExists = con.tableExists(lowerTableName);
-            if (upperExists && lowerExists) {
-                    throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
-                            tableName, upperTableName, lowerTableName));
-            } else if (upperExists) {
-                return upperTableName;
-            } else if (lowerExists) {
-                return lowerTableName;
-            } else {
-                // fallback to the given table name. this may throw error later at getSchemaOfQuery
-                return tableName;
-            }
-        }
-    }
-
-    private List<Integer> findIncrementalColumnIndexes(JdbcSchema schema, List<String> incrementalColumns)
-        throws SQLException
-    {
-        ImmutableList.Builder<Integer> builder = ImmutableList.builder();
-        for (String name : incrementalColumns) {
-            Optional<Integer> index = schema.findColumn(name);
-            if (index.isPresent()) {
-                builder.add(index.get());
-            }
-            else {
-                throw new ConfigException(String.format(ENGLISH,
-                        "Column name '%s' is in incremental_columns option does not exist",
-                        name));
-            }
-        }
-        return builder.build();
-    }
-
-    private String getRawQuery(PluginTask task, JdbcInputConnection con) throws SQLException
-    {
-        if (task.getQuery().isPresent()) {
-            if (task.getTable().isPresent() || task.getSelect().isPresent() ||
-                    task.getWhere().isPresent() || task.getOrderBy().isPresent()) {
-                throw new ConfigException("'table', 'select', 'where' and 'order_by' parameters are unnecessary if 'query' parameter is set.");
-            } else if (!task.getIncrementalColumns().isEmpty() || task.getLastRecord().isPresent()) {
-                throw new ConfigException("'incremental_columns' and 'last_record' parameters are not supported if 'query' parameter is set.");
-            }
-            return task.getQuery().get();
-        } else if (task.getTable().isPresent()) {
-            return con.buildSelectQuery(task.getTable().get(), task.getSelect(),
-                    task.getWhere(), task.getOrderBy());
-        } else {
-            throw new ConfigException("'table' or 'query' parameter is required");
-        }
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control)
-    {
-        PluginTask task = taskSource.loadTask(getTaskClass());
-
-        // TODO when parallel execution is implemented and enabled, (maybe) order_by
-        //      is necessary to resume. transaction() gets the range of order_by
-        //      colum and set it to WHERE condition to make the operation deterministic
-
-        return buildNextConfigDiff(task, control.run(taskSource, schema, taskCount));
-    }
-
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return Exec.newConfigDiff();
-    }
-
-    protected ConfigDiff buildNextConfigDiff(PluginTask task, List<TaskReport> reports)
-    {
-        ConfigDiff next = Exec.newConfigDiff();
-        if (reports.size() > 0 && reports.get(0).has("last_record")) {
-            next.set("last_record", reports.get(0).get(JsonNode.class, "last_record"));
-        } else if (task.getLastRecord().isPresent()) {
-            next.set("last_record", task.getLastRecord().get());
-        }
-        return next;
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    {
-        // do nothing
-    }
-
-    private static class LastRecordStore
-    {
-        private final List<Integer> columnIndexes;
-        private final JsonNode[] lastValues;
-        private final List<String> columnNames;
-
-        public LastRecordStore(List<Integer> columnIndexes, List<String> columnNames)
-        {
-            this.columnIndexes = columnIndexes;
-            this.lastValues = new JsonNode[columnIndexes.size()];
-            this.columnNames = columnNames;
-        }
-
-        public void accept(List<ColumnGetter> getters)
-            throws SQLException
-        {
-            for (int i = 0; i < columnIndexes.size(); i++) {
-                lastValues[i] = getters.get(columnIndexes.get(i)).encodeToJson();
-            }
-        }
-
-        public List<JsonNode> getList()
-        {
-            ImmutableList.Builder<JsonNode> builder = ImmutableList.builder();
-            for (int i = 0; i < lastValues.length; i++) {
-                if (lastValues[i] == null || lastValues[i].isNull()) {
-                    throw new DataException(String.format(ENGLISH,
-                            "incremental_columns can't include null values but the last row is null at column '%s'",
-                            columnNames.get(i)));
-                }
-                builder.add(lastValues[i]);
-            }
-            return builder.build();
-        }
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource,
-            Schema schema, int taskIndex,
-            PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(getTaskClass());
-
-        PreparedQuery builtQuery = task.getBuiltQuery();
-        JdbcSchema querySchema = task.getQuerySchema();
-        BufferAllocator allocator = task.getBufferAllocator();
-        PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
-
-        long totalRows = 0;
-
-        LastRecordStore lastRecordStore = null;
-
-        try (JdbcInputConnection con = newConnection(task)) {
-            List<ColumnGetter> getters = newColumnGetters(con, task, querySchema, pageBuilder);
-            try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
-                while (true) {
-                    long rows = fetch(cursor, getters, pageBuilder);
-                    if (rows <= 0L) {
-                        break;
-                    }
-                    totalRows += rows;
-                }
-            }
-
-            if (task.getIncremental() && totalRows > 0) {
-                lastRecordStore = new LastRecordStore(task.getIncrementalColumnIndexes(), task.getIncrementalColumns());
-                lastRecordStore.accept(getters);
-            }
-
-            pageBuilder.finish();
-
-            // after_select runs after pageBuilder.finish because pageBuilder.finish may fail.
-            // TODO Output plugin's transaction might still fail. In that case, after_select is
-            //      already done but output plugin didn't commit the data to the target storage.
-            //      This means inconsistency between data source and destination. To avoid this
-            //      issue, we need another option like `after_commit` that runs after output plugin's
-            //      commit. after_commit can't run in the same transaction with SELECT. So,
-            //      after_select gets values and store them in TaskReport, and after_commit take
-            //      them as placeholder. Or, after_select puts values to an intermediate table, and
-            //      after_commit moves those values to the actual table.
-            if (task.getAfterSelect().isPresent()) {
-                con.executeUpdate(task.getAfterSelect().get());
-                con.connection.commit();
-            }
-        } catch (SQLException ex) {
-            throw Throwables.propagate(ex);
-        }
-
-        TaskReport report = Exec.newTaskReport();
-        if (lastRecordStore != null) {
-            report.set("last_record", lastRecordStore.getList());
-        }
-
-        return report;
-    }
-
-    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
-    {
-        return new ColumnGetterFactory(pageBuilder, dateTimeZone);
-    }
-
-    private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
-            throws SQLException
-    {
-        ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
-        ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
-        for (JdbcColumn c : querySchema.getColumns()) {
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
-            getters.add(factory.newColumnGetter(con, task, c, columnOption));
-        }
-        return getters.build();
-    }
-
-    private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, Map<String, JdbcColumnOption> defaultColumnOptions, JdbcColumn targetColumn, String targetColumnSQLType)
-    {
-        JdbcColumnOption columnOption = columnOptions.get(targetColumn.getName());
-        if (columnOption == null) {
-            String foundName = null;
-            for (Map.Entry<String, JdbcColumnOption> entry : columnOptions.entrySet()) {
-                if (entry.getKey().equalsIgnoreCase(targetColumn.getName())) {
-                    if (columnOption != null) {
-                        throw new ConfigException(String.format("Cannot specify column '%s' because both '%s' and '%s' exist in column_options.",
-                                targetColumn.getName(), foundName, entry.getKey()));
-                    }
-                    foundName = entry.getKey();
-                    columnOption = entry.getValue();
-                }
-            }
-        }
-
-        return Optional
-                .fromNullable(columnOption)
-                .or(Optional.fromNullable(defaultColumnOptions.get(targetColumnSQLType)))
-                .or(
-                    // default column option
-                    new Supplier<JdbcColumnOption>()
-                    {
-                        public JdbcColumnOption get()
-                        {
-                            return Exec.newConfigSource().loadConfig(JdbcColumnOption.class);
-                        }
-                    });
-    }
-
-    private long fetch(BatchSelect cursor,
-            List<ColumnGetter> getters, PageBuilder pageBuilder) throws SQLException
-    {
-        ResultSet result = cursor.fetch();
-        if (result == null || !result.next()) {
-            return 0;
-        }
-
-        List<Column> columns = pageBuilder.getSchema().getColumns();
-        long rows = 0;
-        long reportRows = 500;
-        do {
-            for (int i=0; i < getters.size(); i++) {
-                int index = i + 1;  // JDBC column index begins from 1
-                getters.get(i).getAndSet(result, index, columns.get(i));
-            }
-            pageBuilder.addRecord();
-            rows++;
-            if (rows % reportRows == 0) {
-                logger.info(String.format("Fetched %,d rows.", rows));
-                reportRows *= 2;
-            }
-        } while (result.next());
-
-        return rows;
-    }
-
-    //// TODO move to embulk.spi.util?
-    //private static class ListPageOutput
-    //{
-    //    public ImmutableList.Builder<Page> pages;
-    //
-    //    public ListPageOutput()
-    //    {
-    //        reset();
-    //    }
-    //
-    //    @Override
-    //    public void add(Page page)
-    //    {
-    //        pages.add(page);
-    //    }
-    //
-    //    @Override
-    //    public void finish()
-    //    {
-    //    }
-    //
-    //    @Override
-    //    public void close()
-    //    {
-    //    }
-    //
-    //    public List<Page> getPages()
-    //    {
-    //        return pages.build();
-    //    }
-    //
-    //    public void reset()
-    //    {
-    //        pages = ImmutableList.builder();
-    //    }
-    //}
-
-    protected void loadDriver(String className, Optional<String> driverPath)
-    {
-        if (driverPath.isPresent()) {
-            addDriverJarToClasspath(driverPath.get());
-        } else {
-            try {
-                // Gradle test task will add JDBC driver to classpath
-                Class.forName(className);
-
-            } catch (ClassNotFoundException ex) {
-                File root = findPluginRoot();
-                File driverLib = new File(root, "default_jdbc_driver");
-                File[] files = driverLib.listFiles(new FileFilter() {
-                    @Override
-                    public boolean accept(File file) {
-                        return file.isFile() && file.getName().endsWith(".jar");
-                    }
-                });
-                if (files == null || files.length == 0) {
-                    throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
-                } else {
-                    for (File file : files) {
-                        logger.info("JDBC Driver = " + file.getAbsolutePath());
-                        addDriverJarToClasspath(file.getAbsolutePath());
-                    }
-                }
-            }
-        }
-
-        // Load JDBC Driver
-        try {
-            Class.forName(className);
-        } catch (ClassNotFoundException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    protected void addDriverJarToClasspath(String glob)
-    {
-        // TODO match glob
-        PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
-        Path path = Paths.get(glob);
-        if (!path.toFile().exists()) {
-             throw new ConfigException("The specified driver jar doesn't exist: " + glob);
-        }
-        loader.addPath(Paths.get(glob));
-    }
-
-    protected File findPluginRoot()
-    {
-        try {
-            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
-            if (url.toString().startsWith("jar:")) {
-                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
-            }
-
-            File folder = new File(url.toURI()).getParentFile();
-            for (;; folder = folder.getParentFile()) {
-                if (folder == null) {
-                    throw new RuntimeException("Cannot find 'embulk-input-xxx' folder.");
-                }
-
-                if (folder.getName().startsWith("embulk-input-")) {
-                    return folder;
-                }
-            }
-        } catch (MalformedURLException | URISyntaxException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    protected void logConnectionProperties(String url, Properties props)
-    {
-        Properties maskedProps = new Properties();
-        for(String key : props.stringPropertyNames()) {
-            if (key.equals("password")) {
-                maskedProps.setProperty(key, "***");
-            } else {
-                maskedProps.setProperty(key, props.getProperty(key));
-            }
-        }
-        logger.info("Connecting to {} options {}", url, maskedProps);
-    }
-}