embulk-input-athena 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3fb336d83e38770353f87f111859738619bb3e6
-  data.tar.gz: 7fe07d1b0fd72c8e08ba963e21b590c26242da79
+  metadata.gz: ad4b3382cf370da50743fdef54e85cbaf9ee3c6b
+  data.tar.gz: 8efd48a28b3dc003aff73c4253abccccdc465df9
 SHA512:
-  metadata.gz: 4483e0985f1741775a6abbf8a9183231be12f52b5f28dc278c850fbf926fba5a2090e9fd660fd6d5b93782dec551b644828e6b767c096254ccc269deb755728f
-  data.tar.gz: bb989515c90ebdd73c651099fcfe6b136d1bfcdc1a5c30c979bbb5f0b3e2d782b8212110cb282f3129888bd347cc6e9878ae5ab2deb4118d108d58201a765f2c
+  metadata.gz: b8dcea7434f50c191421d6bc79d91ab0c0725ead169e86fb40846906e243657cf722d01bba4f9437dfe8e1d40ca901da97583696f42ac8cce268a904038fd1a4
+  data.tar.gz: 69d636d9f53a252db12ff0106e1dfa9b3cfcb00f7c8de005f70137007f850272b54519154dd4db5cf6f3bd35b327dcfddf421389854e6256d1d4aad0358b54f8

data/.gitignore

@@ -10,4 +10,5 @@ build/
 /.metadata/
 .classpath
 .project
-bin
+bin/
+.vscode/

data/build.gradle

@@ -10,12 +10,13 @@ repositories {
     jcenter()
     // for athena jdbc
     maven { url "https://maven.atlassian.com/repository/public" }
+    maven { url "https://dl.bintray.com/embulk-input-jdbc/maven" }
 }
 configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
@@ -24,15 +25,18 @@ dependencies {
     compile  "org.embulk:embulk-core:0.8.39"
     provided "org.embulk:embulk-core:0.8.39"
     // https://mvnrepository.com/artifact/com.amazonaws.athena.jdbc/AthenaJDBC41
-    // compile group: 'com.amazonaws.athena.jdbc', name: 'AthenaJDBC41', version: '1.0.1-atlassian-hosted'
-    compile files ('build/AthenaJDBC41-1.1.0.jar')
+    // TODO: update jdbc
+    compile group: 'com.amazonaws.athena.jdbc', name: 'AthenaJDBC41', version: '1.0.1-atlassian-hosted'
+    //compile files ('build/AthenaJDBC41-1.1.0.jar')
     compile group: 'com.amazonaws', name: 'aws-java-sdk', version: '1.11.301'
+    compile 'org.embulk.input.jdbc:embulk-input-jdbc:0.9.1'
     testCompile "junit:junit:4.+"
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
     from (configurations.runtime - configurations.provided + files(jar.archivePath))
+    // from ("build/AthenaJDBC41-1.1.0.jar'")
     into "classpath"
 }
 clean { delete "classpath" }

data/src/main/java/org/embulk/input/athena/AthenaInputConnection.java

@@ -17,6 +17,7 @@ public class AthenaInputConnection
             throws SQLException
     {
         super(connection, null);
+        connection.setAutoCommit(true);
     }
 
     @Override

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-athena
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - shinji19
@@ -61,33 +61,8 @@ files:
 - src/main/java/org/embulk/input/athena/AthenaInputConnection.java
 - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java
 - src/main/java/org/embulk/input/athena/AthenaInputPlugin.java.tmp1
-- src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java
-- src/main/java/org/embulk/input/jdbc/JdbcColumn.java
-- src/main/java/org/embulk/input/jdbc/JdbcColumnOption.java
-- src/main/java/org/embulk/input/jdbc/JdbcInputConnection.java
-- src/main/java/org/embulk/input/jdbc/JdbcLiteral.java
-- src/main/java/org/embulk/input/jdbc/JdbcSchema.java
-- src/main/java/org/embulk/input/jdbc/Ssl.java
-- src/main/java/org/embulk/input/jdbc/ToString.java
-- src/main/java/org/embulk/input/jdbc/ToStringMap.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractIncrementalHandler.java
-- src/main/java/org/embulk/input/jdbc/getter/AbstractTimestampColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/BigDecimalColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/BooleanColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/ColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/ColumnGetterFactory.java
-- src/main/java/org/embulk/input/jdbc/getter/DateColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/DoubleColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/FloatColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/JsonColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/LongColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/StringColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimeColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampColumnGetter.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampWithTimeZoneIncrementalHandler.java
-- src/main/java/org/embulk/input/jdbc/getter/TimestampWithoutTimeZoneIncrementalHandler.java
 - src/test/java/org/embulk/input/athena/TestAthenaInputPlugin.java
+- classpath/AthenaJDBC41-1.0.1-atlassian-hosted.jar
 - classpath/aws-java-sdk-1.11.301.jar
 - classpath/aws-java-sdk-acm-1.11.301.jar
 - classpath/aws-java-sdk-alexaforbusiness-1.11.301.jar
@@ -217,7 +192,8 @@ files:
 - classpath/aws-java-sdk-xray-1.11.301.jar
 - classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-input-athena-0.1.0.jar
+- classpath/embulk-input-athena-0.1.1.jar
+- classpath/embulk-input-jdbc-0.9.1.jar
 - classpath/httpclient-4.5.5.jar
 - classpath/httpcore-4.4.9.jar
 - classpath/ion-java-1.0.2.jar

data/src/main/java/org/embulk/input/jdbc/AbstractJdbcInputPlugin.java

@@ -1,674 +0,0 @@
-package org.embulk.input.jdbc;
-
-import java.io.File;
-import java.io.FileFilter;
-import java.net.MalformedURLException;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.nio.file.Path;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
-import java.nio.file.Paths;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import org.slf4j.Logger;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.google.common.base.Optional;
-import com.google.common.base.Supplier;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-
-import org.embulk.config.Config;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.plugin.PluginClassLoader;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Column;
-import org.embulk.spi.DataException;
-import org.embulk.spi.PageBuilder;
-import org.embulk.spi.InputPlugin;
-import org.embulk.spi.PageOutput;
-import org.embulk.spi.Schema;
-import org.embulk.spi.Exec;
-import org.embulk.input.jdbc.getter.ColumnGetter;
-import org.embulk.input.jdbc.getter.ColumnGetterFactory;
-import org.embulk.input.jdbc.JdbcInputConnection.BatchSelect;
-import org.embulk.input.jdbc.JdbcInputConnection.PreparedQuery;
-import org.joda.time.DateTimeZone;
-
-import static java.util.Locale.ENGLISH;
-
-public abstract class AbstractJdbcInputPlugin
-        implements InputPlugin
-{
-    protected final Logger logger = Exec.getLogger(getClass());
-
-    public interface PluginTask extends Task
-    {
-        @Config("options")
-        @ConfigDefault("{}")
-        public ToStringMap getOptions();
-
-        @Config("table")
-        @ConfigDefault("null")
-        public Optional<String> getTable();
-        public void setTable(Optional<String> normalizedTableName);
-
-        @Config("query")
-        @ConfigDefault("null")
-        public Optional<String> getQuery();
-
-        @Config("select")
-        @ConfigDefault("null")
-        public Optional<String> getSelect();
-
-        @Config("where")
-        @ConfigDefault("null")
-        public Optional<String> getWhere();
-
-        @Config("order_by")
-        @ConfigDefault("null")
-        public Optional<String> getOrderBy();
-
-        @Config("incremental")
-        @ConfigDefault("false")
-        public boolean getIncremental();
-
-        @Config("incremental_columns")
-        @ConfigDefault("[]")
-        public List<String> getIncrementalColumns();
-        public void setIncrementalColumns(List<String> indexes);
-
-        @Config("last_record")
-        @ConfigDefault("null")
-        public Optional<List<JsonNode>> getLastRecord();
-
-        // TODO limit_value is necessary to make sure repeated bulk load transactions
-        //      don't a same record twice or miss records when the column
-        //      specified at order_by parameter is not unique.
-        //      For example, if the order_by column is "timestamp created_at"
-        //      column whose precision is second, the table can include multiple
-        //      records with the same created_at time. At the first bulk load
-        //      transaction, it loads a record with created_at=2015-01-02 00:00:02.
-        //      Then next transaction will use WHERE created_at > '2015-01-02 00:00:02'.
-        //      However, if another record with created_at=2014-01-01 23:59:59 is
-        //      inserted between the 2 transactions, the new record will be skipped.
-        //      To prevent this scenario, we want to specify
-        //      limit_value=2015-01-02 00:00:00 (exclusive). With this way, as long as
-        //      a transaction runs after 2015-01-02 00:00:00 + some minutes, we don't
-        //      skip records. Ideally, to automate the scheduling, we want to set
-        //      limit_value="today".
-        //
-        //@Config("limit_value")
-        //@ConfigDefault("null")
-        //public Optional<String> getLimitValue();
-
-        //// TODO probably limit_rows is unnecessary as long as this has
-        //        supports parallel execution (partition_by option) and resuming.
-        //@Config("limit_rows")
-        //@ConfigDefault("null")
-        //public Optional<Integer> getLimitRows();
-
-        @Config("connect_timeout")
-        @ConfigDefault("300")
-        public int getConnectTimeout();
-
-        @Config("socket_timeout")
-        @ConfigDefault("1800")
-        public int getSocketTimeout();
-
-        @Config("fetch_rows")
-        @ConfigDefault("10000")
-        // TODO set minimum number
-        public int getFetchRows();
-
-        // TODO parallel execution using "partition_by" config
-
-        @Config("column_options")
-        @ConfigDefault("{}")
-        public Map<String, JdbcColumnOption> getColumnOptions();
-
-        @Config("default_timezone")
-        @ConfigDefault("\"UTC\"")
-        public DateTimeZone getDefaultTimeZone();
-
-        @Config("default_column_options")
-        @ConfigDefault("{}")
-        public Map<String, JdbcColumnOption> getDefaultColumnOptions();
-
-        @Config("after_select")
-        @ConfigDefault("null")
-        public Optional<String> getAfterSelect();
-
-        public PreparedQuery getBuiltQuery();
-        public void setBuiltQuery(PreparedQuery query);
-
-        public JdbcSchema getQuerySchema();
-        public void setQuerySchema(JdbcSchema schema);
-
-        public List<Integer> getIncrementalColumnIndexes();
-        public void setIncrementalColumnIndexes(List<Integer> indexes);
-
-        @ConfigInject
-        public BufferAllocator getBufferAllocator();
-    }
-
-    // for subclasses to add @Config
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return PluginTask.class;
-    }
-
-    protected abstract JdbcInputConnection newConnection(PluginTask task) throws SQLException;
-
-    @Override
-    public ConfigDiff transaction(ConfigSource config,
-            InputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(getTaskClass());
-
-        if (task.getIncremental()) {
-            if (task.getOrderBy().isPresent()) {
-                throw new ConfigException("order_by option must not be set if incremental is true");
-            }
-        }
-        else {
-            if (!task.getIncrementalColumns().isEmpty()) {
-                throw new ConfigException("'incremental: true' must be set if incremental_columns is set");
-            }
-        }
-
-        Schema schema;
-        try (JdbcInputConnection con = newConnection(task)) {
-            con.showDriverVersion();
-
-            // TODO incremental_columns is not set => get primary key
-            schema = setupTask(con, task);
-        } catch (SQLException ex) {
-            throw Throwables.propagate(ex);
-        }
-
-        return buildNextConfigDiff(task, control.run(task.dump(), schema, 1));
-    }
-
-    protected Schema setupTask(JdbcInputConnection con, PluginTask task) throws SQLException
-    {
-        if (task.getTable().isPresent()) {
-            String actualTableName = normalizeTableNameCase(con, task.getTable().get());
-            task.setTable(Optional.of(actualTableName));
-        }
-
-        // build SELECT query and gets schema of its result
-        String rawQuery = getRawQuery(task, con);
-
-        JdbcSchema querySchema = con.getSchemaOfQuery(rawQuery);
-        task.setQuerySchema(querySchema);
-        // query schema should not change after incremental query
-
-        PreparedQuery preparedQuery;
-        if (task.getIncremental()) {
-            // build incremental query
-
-            List<String> incrementalColumns = task.getIncrementalColumns();
-            if (incrementalColumns.isEmpty()) {
-                // incremental_columns is not set
-                if (!task.getTable().isPresent()) {
-                    throw new ConfigException("incremental_columns option must be set if incremental is true and custom query option is set");
-                }
-                // get primary keys from the target table to use them as incremental_columns
-                List<String> primaryKeys = con.getPrimaryKeys(task.getTable().get());
-                if (primaryKeys.isEmpty()) {
-                    throw new ConfigException(String.format(ENGLISH,
-                                "Primary key is not available at the table '%s'. incremental_columns option must be set",
-                                task.getTable().get()));
-                }
-                logger.info("Using primary keys as incremental_columns: {}", primaryKeys);
-                task.setIncrementalColumns(primaryKeys);
-                incrementalColumns = primaryKeys;
-            }
-
-            List<Integer> incrementalColumnIndexes = findIncrementalColumnIndexes(querySchema, incrementalColumns);
-            task.setIncrementalColumnIndexes(incrementalColumnIndexes);
-
-            List<JsonNode> lastRecord;
-            if (task.getLastRecord().isPresent()) {
-                lastRecord = task.getLastRecord().get();
-                if (lastRecord.size() != incrementalColumnIndexes.size()) {
-                    throw new ConfigException("Number of values set at last_record must be same with number of columns set at incremental_columns");
-                }
-            }
-            else {
-                lastRecord = null;
-            }
-
-            if (task.getQuery().isPresent()) {
-                preparedQuery = con.wrapIncrementalQuery(rawQuery, querySchema, incrementalColumnIndexes, lastRecord);
-            }
-            else {
-                preparedQuery = con.rebuildIncrementalQuery(
-                        task.getTable().get(), task.getSelect(),
-                        task.getWhere(),
-                        querySchema, incrementalColumnIndexes, lastRecord);
-            }
-        }
-        else {
-            task.setIncrementalColumnIndexes(ImmutableList.<Integer>of());
-            preparedQuery = new PreparedQuery(rawQuery, ImmutableList.<JdbcLiteral>of());
-        }
-
-        task.setBuiltQuery(preparedQuery);
-
-        // validate column_options
-        newColumnGetters(con, task, querySchema, null);
-
-        ColumnGetterFactory factory = newColumnGetterFactory(null, task.getDefaultTimeZone());
-        ImmutableList.Builder<Column> columns = ImmutableList.builder();
-        for (int i = 0; i < querySchema.getCount(); i++) {
-            JdbcColumn column = querySchema.getColumn(i);
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), column, factory.getJdbcType(column.getSqlType()));
-            columns.add(new Column(i,
-                    column.getName(),
-                    factory.newColumnGetter(con, task, column, columnOption).getToType()));
-        }
-        return new Schema(columns.build());
-    }
-
-    private String normalizeTableNameCase(JdbcInputConnection con, String tableName)
-        throws SQLException
-    {
-        if (con.tableExists(tableName)) {
-            return tableName;
-        } else {
-            String upperTableName = tableName.toUpperCase();
-            String lowerTableName = tableName.toLowerCase();
-            boolean upperExists = con.tableExists(upperTableName);
-            boolean lowerExists = con.tableExists(lowerTableName);
-            if (upperExists && lowerExists) {
-                    throw new ConfigException(String.format("Cannot specify table '%s' because both '%s' and '%s' exist.",
-                            tableName, upperTableName, lowerTableName));
-            } else if (upperExists) {
-                return upperTableName;
-            } else if (lowerExists) {
-                return lowerTableName;
-            } else {
-                // fallback to the given table name. this may throw error later at getSchemaOfQuery
-                return tableName;
-            }
-        }
-    }
-
-    private List<Integer> findIncrementalColumnIndexes(JdbcSchema schema, List<String> incrementalColumns)
-        throws SQLException
-    {
-        ImmutableList.Builder<Integer> builder = ImmutableList.builder();
-        for (String name : incrementalColumns) {
-            Optional<Integer> index = schema.findColumn(name);
-            if (index.isPresent()) {
-                builder.add(index.get());
-            }
-            else {
-                throw new ConfigException(String.format(ENGLISH,
-                        "Column name '%s' is in incremental_columns option does not exist",
-                        name));
-            }
-        }
-        return builder.build();
-    }
-
-    private String getRawQuery(PluginTask task, JdbcInputConnection con) throws SQLException
-    {
-        if (task.getQuery().isPresent()) {
-            if (task.getTable().isPresent() || task.getSelect().isPresent() ||
-                    task.getWhere().isPresent() || task.getOrderBy().isPresent()) {
-                throw new ConfigException("'table', 'select', 'where' and 'order_by' parameters are unnecessary if 'query' parameter is set.");
-            } else if (!task.getIncrementalColumns().isEmpty() || task.getLastRecord().isPresent()) {
-                throw new ConfigException("'incremental_columns' and 'last_record' parameters are not supported if 'query' parameter is set.");
-            }
-            return task.getQuery().get();
-        } else if (task.getTable().isPresent()) {
-            return con.buildSelectQuery(task.getTable().get(), task.getSelect(),
-                    task.getWhere(), task.getOrderBy());
-        } else {
-            throw new ConfigException("'table' or 'query' parameter is required");
-        }
-    }
-
-    @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            InputPlugin.Control control)
-    {
-        PluginTask task = taskSource.loadTask(getTaskClass());
-
-        // TODO when parallel execution is implemented and enabled, (maybe) order_by
-        //      is necessary to resume. transaction() gets the range of order_by
-        //      colum and set it to WHERE condition to make the operation deterministic
-
-        return buildNextConfigDiff(task, control.run(taskSource, schema, taskCount));
-    }
-
-    public ConfigDiff guess(ConfigSource config)
-    {
-        return Exec.newConfigDiff();
-    }
-
-    protected ConfigDiff buildNextConfigDiff(PluginTask task, List<TaskReport> reports)
-    {
-        ConfigDiff next = Exec.newConfigDiff();
-        if (reports.size() > 0 && reports.get(0).has("last_record")) {
-            next.set("last_record", reports.get(0).get(JsonNode.class, "last_record"));
-        } else if (task.getLastRecord().isPresent()) {
-            next.set("last_record", task.getLastRecord().get());
-        }
-        return next;
-    }
-
-    @Override
-    public void cleanup(TaskSource taskSource,
-            Schema schema, int taskCount,
-            List<TaskReport> successTaskReports)
-    {
-        // do nothing
-    }
-
-    private static class LastRecordStore
-    {
-        private final List<Integer> columnIndexes;
-        private final JsonNode[] lastValues;
-        private final List<String> columnNames;
-
-        public LastRecordStore(List<Integer> columnIndexes, List<String> columnNames)
-        {
-            this.columnIndexes = columnIndexes;
-            this.lastValues = new JsonNode[columnIndexes.size()];
-            this.columnNames = columnNames;
-        }
-
-        public void accept(List<ColumnGetter> getters)
-            throws SQLException
-        {
-            for (int i = 0; i < columnIndexes.size(); i++) {
-                lastValues[i] = getters.get(columnIndexes.get(i)).encodeToJson();
-            }
-        }
-
-        public List<JsonNode> getList()
-        {
-            ImmutableList.Builder<JsonNode> builder = ImmutableList.builder();
-            for (int i = 0; i < lastValues.length; i++) {
-                if (lastValues[i] == null || lastValues[i].isNull()) {
-                    throw new DataException(String.format(ENGLISH,
-                            "incremental_columns can't include null values but the last row is null at column '%s'",
-                            columnNames.get(i)));
-                }
-                builder.add(lastValues[i]);
-            }
-            return builder.build();
-        }
-    }
-
-    @Override
-    public TaskReport run(TaskSource taskSource,
-            Schema schema, int taskIndex,
-            PageOutput output)
-    {
-        PluginTask task = taskSource.loadTask(getTaskClass());
-
-        PreparedQuery builtQuery = task.getBuiltQuery();
-        JdbcSchema querySchema = task.getQuerySchema();
-        BufferAllocator allocator = task.getBufferAllocator();
-        PageBuilder pageBuilder = new PageBuilder(allocator, schema, output);
-
-        long totalRows = 0;
-
-        LastRecordStore lastRecordStore = null;
-
-        try (JdbcInputConnection con = newConnection(task)) {
-            List<ColumnGetter> getters = newColumnGetters(con, task, querySchema, pageBuilder);
-            try (BatchSelect cursor = con.newSelectCursor(builtQuery, getters, task.getFetchRows(), task.getSocketTimeout())) {
-                while (true) {
-                    long rows = fetch(cursor, getters, pageBuilder);
-                    if (rows <= 0L) {
-                        break;
-                    }
-                    totalRows += rows;
-                }
-            }
-
-            if (task.getIncremental() && totalRows > 0) {
-                lastRecordStore = new LastRecordStore(task.getIncrementalColumnIndexes(), task.getIncrementalColumns());
-                lastRecordStore.accept(getters);
-            }
-
-            pageBuilder.finish();
-
-            // after_select runs after pageBuilder.finish because pageBuilder.finish may fail.
-            // TODO Output plugin's transaction might still fail. In that case, after_select is
-            //      already done but output plugin didn't commit the data to the target storage.
-            //      This means inconsistency between data source and destination. To avoid this
-            //      issue, we need another option like `after_commit` that runs after output plugin's
-            //      commit. after_commit can't run in the same transaction with SELECT. So,
-            //      after_select gets values and store them in TaskReport, and after_commit take
-            //      them as placeholder. Or, after_select puts values to an intermediate table, and
-            //      after_commit moves those values to the actual table.
-            if (task.getAfterSelect().isPresent()) {
-                con.executeUpdate(task.getAfterSelect().get());
-                con.connection.commit();
-            }
-        } catch (SQLException ex) {
-            throw Throwables.propagate(ex);
-        }
-
-        TaskReport report = Exec.newTaskReport();
-        if (lastRecordStore != null) {
-            report.set("last_record", lastRecordStore.getList());
-        }
-
-        return report;
-    }
-
-    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
-    {
-        return new ColumnGetterFactory(pageBuilder, dateTimeZone);
-    }
-
-    private List<ColumnGetter> newColumnGetters(JdbcInputConnection con, PluginTask task, JdbcSchema querySchema, PageBuilder pageBuilder)
-            throws SQLException
-    {
-        ColumnGetterFactory factory = newColumnGetterFactory(pageBuilder, task.getDefaultTimeZone());
-        ImmutableList.Builder<ColumnGetter> getters = ImmutableList.builder();
-        for (JdbcColumn c : querySchema.getColumns()) {
-            JdbcColumnOption columnOption = columnOptionOf(task.getColumnOptions(), task.getDefaultColumnOptions(), c, factory.getJdbcType(c.getSqlType()));
-            getters.add(factory.newColumnGetter(con, task, c, columnOption));
-        }
-        return getters.build();
-    }
-
-    private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, Map<String, JdbcColumnOption> defaultColumnOptions, JdbcColumn targetColumn, String targetColumnSQLType)
-    {
-        JdbcColumnOption columnOption = columnOptions.get(targetColumn.getName());
-        if (columnOption == null) {
-            String foundName = null;
-            for (Map.Entry<String, JdbcColumnOption> entry : columnOptions.entrySet()) {
-                if (entry.getKey().equalsIgnoreCase(targetColumn.getName())) {
-                    if (columnOption != null) {
-                        throw new ConfigException(String.format("Cannot specify column '%s' because both '%s' and '%s' exist in column_options.",
-                                targetColumn.getName(), foundName, entry.getKey()));
-                    }
-                    foundName = entry.getKey();
-                    columnOption = entry.getValue();
-                }
-            }
-        }
-
-        return Optional
-                .fromNullable(columnOption)
-                .or(Optional.fromNullable(defaultColumnOptions.get(targetColumnSQLType)))
-                .or(
-                    // default column option
-                    new Supplier<JdbcColumnOption>()
-                    {
-                        public JdbcColumnOption get()
-                        {
-                            return Exec.newConfigSource().loadConfig(JdbcColumnOption.class);
-                        }
-                    });
-    }
-
-    private long fetch(BatchSelect cursor,
-            List<ColumnGetter> getters, PageBuilder pageBuilder) throws SQLException
-    {
-        ResultSet result = cursor.fetch();
-        if (result == null || !result.next()) {
-            return 0;
-        }
-
-        List<Column> columns = pageBuilder.getSchema().getColumns();
-        long rows = 0;
-        long reportRows = 500;
-        do {
-            for (int i=0; i < getters.size(); i++) {
-                int index = i + 1;  // JDBC column index begins from 1
-                getters.get(i).getAndSet(result, index, columns.get(i));
-            }
-            pageBuilder.addRecord();
-            rows++;
-            if (rows % reportRows == 0) {
-                logger.info(String.format("Fetched %,d rows.", rows));
-                reportRows *= 2;
-            }
-        } while (result.next());
-
-        return rows;
-    }
-
-    //// TODO move to embulk.spi.util?
-    //private static class ListPageOutput
-    //{
-    //    public ImmutableList.Builder<Page> pages;
-    //
-    //    public ListPageOutput()
-    //    {
-    //        reset();
-    //    }
-    //
-    //    @Override
-    //    public void add(Page page)
-    //    {
-    //        pages.add(page);
-    //    }
-    //
-    //    @Override
-    //    public void finish()
-    //    {
-    //    }
-    //
-    //    @Override
-    //    public void close()
-    //    {
-    //    }
-    //
-    //    public List<Page> getPages()
-    //    {
-    //        return pages.build();
-    //    }
-    //
-    //    public void reset()
-    //    {
-    //        pages = ImmutableList.builder();
-    //    }
-    //}
-
-    protected void loadDriver(String className, Optional<String> driverPath)
-    {
-        if (driverPath.isPresent()) {
-            addDriverJarToClasspath(driverPath.get());
-        } else {
-            try {
-                // Gradle test task will add JDBC driver to classpath
-                Class.forName(className);
-
-            } catch (ClassNotFoundException ex) {
-                File root = findPluginRoot();
-                File driverLib = new File(root, "default_jdbc_driver");
-                File[] files = driverLib.listFiles(new FileFilter() {
-                    @Override
-                    public boolean accept(File file) {
-                        return file.isFile() && file.getName().endsWith(".jar");
-                    }
-                });
-                if (files == null || files.length == 0) {
-                    throw new RuntimeException("Cannot find JDBC driver in '" + root.getAbsolutePath() + "'.");
-                } else {
-                    for (File file : files) {
-                        logger.info("JDBC Driver = " + file.getAbsolutePath());
-                        addDriverJarToClasspath(file.getAbsolutePath());
-                    }
-                }
-            }
-        }
-
-        // Load JDBC Driver
-        try {
-            Class.forName(className);
-        } catch (ClassNotFoundException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
-    protected void addDriverJarToClasspath(String glob)
-    {
-        // TODO match glob
-        PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
-        Path path = Paths.get(glob);
-        if (!path.toFile().exists()) {
-             throw new ConfigException("The specified driver jar doesn't exist: " + glob);
-        }
-        loader.addPath(Paths.get(glob));
-    }
-
-    protected File findPluginRoot()
-    {
-        try {
-            URL url = getClass().getResource("/" + getClass().getName().replace('.', '/') + ".class");
-            if (url.toString().startsWith("jar:")) {
-                url = new URL(url.toString().replaceAll("^jar:", "").replaceAll("![^!]*$", ""));
-            }
-
-            File folder = new File(url.toURI()).getParentFile();
-            for (;; folder = folder.getParentFile()) {
-                if (folder == null) {
-                    throw new RuntimeException("Cannot find 'embulk-input-xxx' folder.");
-                }
-
-                if (folder.getName().startsWith("embulk-input-")) {
-                    return folder;
-                }
-            }
-        } catch (MalformedURLException | URISyntaxException e) {
-            throw new RuntimeException(e);
-        }
-    }
-
-    protected void logConnectionProperties(String url, Properties props)
-    {
-        Properties maskedProps = new Properties();
-        for(String key : props.stringPropertyNames()) {
-            if (key.equals("password")) {
-                maskedProps.setProperty(key, "***");
-            } else {
-                maskedProps.setProperty(key, props.getProperty(key));
-            }
-        }
-        logger.info("Connecting to {} options {}", url, maskedProps);
-    }
-}