embulk-output-jdbc 0.2.4 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a3248904e8b007fc46013b4697d939d7dd3f55d8
-  data.tar.gz: f3eacb5f104d9ea4f474e14a7c0ea724c9831341
+  metadata.gz: adf000cfbd28d4fb1c8e33f8c11f9551b34c8c70
+  data.tar.gz: dc37c2dd32bc03ff6b92d16a57da20614bd0dd25
 SHA512:
-  metadata.gz: 5d8f93e2f5f014df77b5675ce58209e1c17a43b02e17a4dbe5fc57348461a90659a5aacd8c61f8f68bba94c08cab0da2facd6989c57b90e8357c810d2c3f84be
-  data.tar.gz: 6afcba3953c431e6c0fc4e5733a91c4dff9b4c04af6d5eb04869aeb7b8c2d432f2df8ddc6a96ae725bfeae18cb62f413be206f9dcad2246391de45cbfbe87c5b
+  metadata.gz: b5768c7090977ee2f5d01ca59808ea6eea7698d07cea80daf29d06617f13a2befc4e7354de954019590f2bd1c2813382ddb4f3ba97bfdb7713906d92822334e6
+  data.tar.gz: 8eea4f3829f024e86e661324edd4b948be3a73725826d94b426c3b08ab760c464496349cad2c4948cda09f5f1abb58e76d03b65bf31fe3f90f0221e5596611c9

data/build.gradle

@@ -1,2 +1,2 @@
-dependencies {
-}
+dependencies {
+}

data/lib/embulk/output/jdbc.rb

@@ -1,3 +1,3 @@
-Embulk::JavaPlugin.register_output(
-  :jdbc, "org.embulk.output.JdbcOutputPlugin",
-  File.expand_path('../../../../classpath', __FILE__))
+Embulk::JavaPlugin.register_output(
+  :jdbc, "org.embulk.output.JdbcOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/JdbcOutputPlugin.java

@@ -1,120 +1,138 @@
-package org.embulk.output;
-
-import java.util.Properties;
-import java.sql.Driver;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.SQLException;
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
-import org.embulk.output.jdbc.BatchInsert;
-import org.embulk.output.jdbc.StandardBatchInsert;
-import org.embulk.output.jdbc.JdbcOutputConnector;
-import org.embulk.output.jdbc.JdbcOutputConnection;
-
-public class JdbcOutputPlugin
-        extends AbstractJdbcOutputPlugin
-{
-    public interface GenericPluginTask extends PluginTask
-    {
-        @Config("driver_path")
-        @ConfigDefault("null")
-        public Optional<String> getDriverPath();
-
-        @Config("driver_class")
-        public String getDriverClass();
-
-        @Config("url")
-        public String getUrl();
-
-        @Config("user")
-        @ConfigDefault("null")
-        public Optional<String> getUser();
-
-        @Config("password")
-        @ConfigDefault("null")
-        public Optional<String> getPassword();
-
-        @Config("schema")
-        @ConfigDefault("null")
-        public Optional<String> getSchema();
-    }
-
-    @Override
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return GenericPluginTask.class;
-    }
-
-    @Override
-    protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
-    {
-        GenericPluginTask t = (GenericPluginTask) task;
-
-        if (t.getDriverPath().isPresent()) {
-            loadDriverJar(t.getDriverPath().get());
-        }
-
-        Properties props = new Properties();
-        if (t.getUser().isPresent()) {
-            props.setProperty("user", t.getUser().get());
-        }
-        if (t.getPassword().isPresent()) {
-            props.setProperty("password", t.getPassword().get());
-        }
-
-        props.putAll(t.getOptions());
-
-        return new GenericOutputConnector(t.getUrl(), props, t.getDriverClass(),
-                t.getSchema().orNull());
-    }
-
-    private static class GenericOutputConnector
-            implements JdbcOutputConnector
-    {
-        private final Driver driver;
-        private final String url;
-        private final Properties properties;
-        private final String schemaName;
-
-        public GenericOutputConnector(String url, Properties properties, String driverClass,
-                String schemaName)
-        {
-            try {
-                // TODO check Class.forName(driverClass) is a Driver before newInstance
-                //      for security
-                this.driver = (Driver) Class.forName(driverClass).newInstance();
-            } catch (Exception ex) {
-                throw Throwables.propagate(ex);
-            }
-            this.url = url;
-            this.properties = properties;
-            this.schemaName = schemaName;
-        }
-
-        @Override
-        public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
-        {
-            Connection c = driver.connect(url, properties);
-            try {
-                c.setAutoCommit(autoCommit);
-                JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
-                c = null;
-                return con;
-            } finally {
-                if (c != null) {
-                    c.close();
-                }
-            }
-        }
-    }
-
-    @Override
-    protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
-    {
-        return new StandardBatchInsert(getConnector(task, true));
-    }
-}
+package org.embulk.output;
+
+import java.util.List;
+import java.util.Set;
+import java.util.Properties;
+import java.sql.Driver;
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import com.google.common.base.Optional;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableSet;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.StandardBatchInsert;
+import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+
+public class JdbcOutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    public interface GenericPluginTask extends PluginTask
+    {
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
+        @Config("driver_class")
+        public String getDriverClass();
+
+        @Config("url")
+        public String getUrl();
+
+        @Config("user")
+        @ConfigDefault("null")
+        public Optional<String> getUser();
+
+        @Config("password")
+        @ConfigDefault("null")
+        public Optional<String> getPassword();
+
+        @Config("schema")
+        @ConfigDefault("null")
+        public Optional<String> getSchema();
+
+        @Config("max_table_name_length")
+        @ConfigDefault("30")
+        public int getMaxTableNameLength();
+    }
+
+    @Override
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return GenericPluginTask.class;
+    }
+
+    @Override
+    protected Features getFeatures(PluginTask task)
+    {
+        GenericPluginTask t = (GenericPluginTask) task;
+        return new Features()
+            .setMaxTableNameLength(t.getMaxTableNameLength())
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE));
+    }
+
+    @Override
+    protected GenericOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        GenericPluginTask t = (GenericPluginTask) task;
+
+        if (t.getDriverPath().isPresent()) {
+            loadDriverJar(t.getDriverPath().get());
+        }
+
+        Properties props = new Properties();
+
+        props.putAll(t.getOptions());
+
+        if (t.getUser().isPresent()) {
+            props.setProperty("user", t.getUser().get());
+        }
+        logger.info("Connecting to {} options {}", t.getUrl(), props);
+        if (t.getPassword().isPresent()) {
+            props.setProperty("password", t.getPassword().get());
+        }
+
+        return new GenericOutputConnector(t.getUrl(), props, t.getDriverClass(),
+                t.getSchema().orNull());
+    }
+
+    private static class GenericOutputConnector
+            implements JdbcOutputConnector
+    {
+        private final Driver driver;
+        private final String url;
+        private final Properties properties;
+        private final String schemaName;
+
+        public GenericOutputConnector(String url, Properties properties, String driverClass,
+                String schemaName)
+        {
+            try {
+                // TODO check Class.forName(driverClass) is a Driver before newInstance
+                //      for security
+                this.driver = (Driver) Class.forName(driverClass).newInstance();
+            } catch (Exception ex) {
+                throw Throwables.propagate(ex);
+            }
+            this.url = url;
+            this.properties = properties;
+            this.schemaName = schemaName;
+        }
+
+        @Override
+        public JdbcOutputConnection connect(boolean autoCommit) throws SQLException
+        {
+            Connection c = driver.connect(url, properties);
+            try {
+                c.setAutoCommit(autoCommit);
+                JdbcOutputConnection con = new JdbcOutputConnection(c, schemaName);
+                c = null;
+                return con;
+            } finally {
+                if (c != null) {
+                    c.close();
+                }
+            }
+        }
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    {
+        return new StandardBatchInsert(getConnector(task, true), mergeKeys);
+    }
+}

data/src/main/java/org/embulk/output/jdbc/AbstractJdbcOutputPlugin.java

@@ -1,755 +1,964 @@
-package org.embulk.output.jdbc;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Locale;
-import java.util.Properties;
-import java.util.Set;
-import java.util.concurrent.ExecutionException;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.sql.Types;
-import java.sql.ResultSet;
-import java.sql.DatabaseMetaData;
-import java.sql.SQLException;
-
-import org.slf4j.Logger;
-
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
-import com.google.common.collect.ImmutableList;
-
-import org.embulk.config.CommitReport;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigException;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskSource;
-import org.embulk.spi.Exec;
-import org.embulk.spi.Column;
-import org.embulk.spi.ColumnVisitor;
-import org.embulk.spi.OutputPlugin;
-import org.embulk.spi.PluginClassLoader;
-import org.embulk.spi.Schema;
-import org.embulk.spi.TransactionalPageOutput;
-import org.embulk.spi.Page;
-import org.embulk.spi.PageReader;
-import org.embulk.spi.time.Timestamp;
-import org.embulk.spi.time.TimestampFormatter;
-import org.embulk.output.jdbc.setter.ColumnSetter;
-import org.embulk.output.jdbc.setter.ColumnSetterFactory;
-import org.embulk.output.jdbc.RetryExecutor.IdempotentOperation;
-
-import static org.embulk.output.jdbc.RetryExecutor.retryExecutor;
-
-public abstract class AbstractJdbcOutputPlugin
-        implements OutputPlugin
-{
-    private final static Set<String> loadedJarGlobs = new HashSet<String>();
-
-    private final Logger logger = Exec.getLogger(getClass());
-
-    public interface PluginTask
-            extends Task
-    {
-        @Config("options")
-        @ConfigDefault("{}")
-        public Properties getOptions();
-
-        @Config("table")
-        public String getTable();
-
-        @Config("mode")
-        public String getModeConfig();
-
-        @Config("batch_size")
-        @ConfigDefault("16777216")
-        // TODO set minimum number
-        public int getBatchSize();
-
-        public void setMode(Mode mode);
-        public Mode getMode();
-
-        public JdbcSchema getLoadSchema();
-        public void setLoadSchema(JdbcSchema schema);
-
-        public Optional<String> getSwapTable();
-        public void setSwapTable(Optional<String> name);
-
-        public Optional<String> getMultipleLoadTablePrefix();
-        public void setMultipleLoadTablePrefix(Optional<String> prefix);
-    }
-
-    protected void loadDriverJar(String glob)
-    {
-        synchronized (loadedJarGlobs) {
-            if (!loadedJarGlobs.contains(glob)) {
-                // TODO match glob
-                PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
-                loader.addPath(Paths.get(glob));
-                loadedJarGlobs.add(glob);
-            }
-        }
-    }
-
-    // for subclasses to add @Config
-    protected Class<? extends PluginTask> getTaskClass()
-    {
-        return PluginTask.class;
-    }
-
-    protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
-
-    protected abstract BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException;
-
-    protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
-            boolean autoCommit) throws SQLException
-    {
-        return getConnector(task, retryableMetadataOperation).connect(autoCommit);
-    }
-
-    public enum Mode {
-        INSERT,
-        INSERT_DIRECT,
-        TRUNCATE_INSERT,
-        MERGE,
-        REPLACE,
-        REPLACE_INPLACE;
-        //REPLACE_PARTITIONING,  // MySQL: partitioning, PostgreSQL: inheritance
-
-        public boolean isDirectWrite()
-        {
-            return this == INSERT_DIRECT;
-        }
-
-        public boolean isInplace()
-        {
-            return this == INSERT_DIRECT || this == REPLACE_INPLACE || this == MERGE;
-        }
-
-        public boolean isMerge()
-        {
-            return this == MERGE;
-        }
-
-        public boolean usesMultipleLoadTables()
-        {
-            return !isInplace();
-        }
-
-        public boolean createAndSwapTable()
-        {
-            return this == REPLACE_INPLACE || this == REPLACE;
-        }
-    }
-
-    public ConfigDiff transaction(ConfigSource config,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control)
-    {
-        PluginTask task = config.loadConfig(getTaskClass());
-
-        // TODO this is a temporary code. behavior will change in a future release.
-        switch(task.getModeConfig()) {
-        case "insert":
-            task.setMode(Mode.INSERT_DIRECT);
-            break;
-        case "replace":
-            task.setMode(Mode.REPLACE_INPLACE);
-            break;
-        case "merge":
-            task.setMode(Mode.MERGE);
-            break;
-        default:
-            throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert, replace, merge", task.getModeConfig()));
-        }
-
-        //switch(task.getModeConfig()) {
-        ////case "insert":
-        ////    task.setMode(Mode.INSERT);
-        ////    break;
-        //case "insert_direct":
-        //    task.setMode(Mode.INSERT_DIRECT);
-        //    break;
-        ////case "truncate_insert":  // TODO
-        ////    task.setMode(Mode.TRUNCATE_INSERT);
-        ////    break;
-        ////case "merge":  // TODO
-        ////    task.setMode(Mode.MERGE);
-        ////    break;
-        ////case "replace":
-        ////    task.setMode(Mode.REPLACE);
-        ////    break;
-        //case "replace_inplace":
-        //    task.setMode(Mode.REPLACE_INPLACE);
-        //    break;
-        //default:
-        //    new ConfigException(String.format("Unknown mode '%s'. Supported modes are: insert_direct, replace_inplace", task.getModeConfig()));
-        //}
-
-        task = begin(task, schema, taskCount);
-        control.run(task.dump());
-        return commit(task, schema, taskCount);
-    }
-
-    public ConfigDiff resume(TaskSource taskSource,
-            Schema schema, int taskCount,
-            OutputPlugin.Control control)
-    {
-        PluginTask task = taskSource.loadTask(getTaskClass());
-
-        if (task.getMode().isInplace()) {
-            throw new UnsupportedOperationException("inplace mode is not resumable. You need to delete partially-loaded records from the database and restart the entire transaction.");
-        }
-
-        task = begin(task, schema, taskCount);
-        control.run(task.dump());
-        return commit(task, schema, taskCount);
-    }
-
-    protected String getTransactionUniqueName()
-    {
-        // TODO use uuid?
-        Timestamp t = Exec.session().getTransactionTime();
-        return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
-    }
-
-    private PluginTask begin(final PluginTask task,
-            final Schema schema, int taskCount)
-    {
-        try {
-            withRetry(new IdempotentSqlRunnable() {  // no intermediate data if isDirectWrite == true
-                public void run() throws SQLException
-                {
-                    JdbcOutputConnection con = newConnection(task, true, false);
-                    try {
-                        doBegin(con, task, schema);
-                    } finally {
-                        con.close();
-                    }
-                }
-            });
-        } catch (SQLException | InterruptedException ex) {
-            throw new RuntimeException(ex);
-        }
-        return task;
-    }
-
-    private ConfigDiff commit(final PluginTask task,
-            Schema schema, final int taskCount)
-    {
-        if (!task.getMode().isDirectWrite()) {  // no intermediate data if isDirectWrite == true
-            try {
-                withRetry(new IdempotentSqlRunnable() {
-                    public void run() throws SQLException
-                    {
-                        JdbcOutputConnection con = newConnection(task, false, false);
-                        try {
-                            doCommit(con, task, taskCount);
-                        } finally {
-                            con.close();
-                        }
-                    }
-                });
-            } catch (SQLException | InterruptedException ex) {
-                throw new RuntimeException(ex);
-            }
-        }
-        return Exec.newConfigDiff();
-    }
-
-    public void cleanup(TaskSource taskSource,
-            Schema schema, final int taskCount,
-            final List<CommitReport> successCommitReports)
-    {
-        final PluginTask task = taskSource.loadTask(getTaskClass());
-
-        if (!task.getMode().isDirectWrite()) {  // no intermediate data if isDirectWrite == true
-            try {
-                withRetry(new IdempotentSqlRunnable() {
-                    public void run() throws SQLException
-                    {
-                        JdbcOutputConnection con = newConnection(task, true, true);
-                        try {
-                            doCleanup(con, task, taskCount, successCommitReports);
-                        } finally {
-                            con.close();
-                        }
-                    }
-                });
-            } catch (SQLException | InterruptedException ex) {
-                throw new RuntimeException(ex);
-            }
-        }
-    }
-
-    protected void doBegin(JdbcOutputConnection con,
-            PluginTask task, Schema schema) throws SQLException
-    {
-        Mode mode = task.getMode();
-
-        JdbcSchema targetTableSchema;
-        if (mode.createAndSwapTable()) {
-            // DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
-            // CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
-            // swapTableName = "xyz__0000000054d92dee1e452158_bulk_load_temp"
-            String swapTableName = generateSwapTableName(task);
-            con.dropTableIfExists(swapTableName);
-            con.createTableIfNotExists(swapTableName, newJdbcSchemaForNewTable(schema));
-            targetTableSchema = newJdbcSchemaFromExistentTable(con, swapTableName);
-            task.setSwapTable(Optional.of(swapTableName));
-        } else {
-            // CREATE TABLE IF NOT EXISTS xyz
-            con.createTableIfNotExists(task.getTable(), newJdbcSchemaForNewTable(schema));
-            targetTableSchema = newJdbcSchemaFromExistentTable(con, task.getTable());
-            task.setSwapTable(Optional.<String>absent());
-        }
-
-        if (mode.usesMultipleLoadTables()) {
-            // multipleLoadTablePrefix = "xyz__0000000054d92dee1e452158_"
-            // workers run:
-            //   CREATE TABLE xyz__0000000054d92dee1e452158_%d
-            String multipleLoadTablePrefix = task.getTable() + "_" + getTransactionUniqueName();
-            task.setMultipleLoadTablePrefix(Optional.of(multipleLoadTablePrefix));
-        } else {
-            task.setMultipleLoadTablePrefix(Optional.<String>absent());
-        }
-
-        task.setLoadSchema(matchSchemaByColumnNames(schema, targetTableSchema));
-    }
-
-    protected String generateSwapTableName(PluginTask task) throws SQLException
-    {
-    	return task.getTable() + "_" + getTransactionUniqueName() + "_bulk_load_temp";
-    }
-
-    protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
-        throws SQLException
-    {
-        switch (task.getMode()) {
-        case INSERT:
-            // aggregate insert into target
-            //con.gatherInsertTables();
-            throw new UnsupportedOperationException("not implemented yet"); // TODO
-        case INSERT_DIRECT:
-            // already done
-            break;
-        case TRUNCATE_INSERT:
-            // truncate & aggregate insert into target
-            throw new UnsupportedOperationException("not implemented yet");
-            //break;
-        case MERGE:
-            // aggregate merge into target
-            throw new UnsupportedOperationException("not implemented yet");
-            //break;
-        case REPLACE:
-            if (taskCount == 1) {
-                // swap table
-                con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
-            } else {
-                // aggregate insert into swap table & swap table
-                throw new UnsupportedOperationException("not implemented yet");
-            }
-            break;
-        case REPLACE_INPLACE:
-            // swap table
-            con.replaceTable(task.getSwapTable().get(), task.getLoadSchema(), task.getTable());
-            break;
-        }
-    }
-
-    protected void doCleanup(JdbcOutputConnection con, PluginTask task, int taskCount,
-            List<CommitReport> successCommitReports)
-        throws SQLException
-    {
-        if (task.getSwapTable().isPresent()) {
-            con.dropTableIfExists(task.getSwapTable().get());
-        }
-        if (task.getMultipleLoadTablePrefix().isPresent()) {
-            for (int i=0; i < taskCount; i++) {
-                con.dropTableIfExists(formatMultipleLoadTableName(task, i));
-            }
-        }
-    }
-
-    static String formatMultipleLoadTableName(PluginTask task, int taskIndex)
-    {
-        return task.getMultipleLoadTablePrefix().get() + String.format("%04x", taskIndex);
-    }
-
-    protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
-    {
-        final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
-        for (Column c : schema.getColumns()) {
-            final String columnName = c.getName();
-            c.visit(new ColumnVisitor() {
-                public void booleanColumn(Column column)
-                {
-                    columns.add(new JdbcColumn(
-                            columnName, "BOOLEAN",
-                            Types.BOOLEAN, 1, 0, false));
-                }
-
-                public void longColumn(Column column)
-                {
-                    columns.add(new JdbcColumn(
-                            columnName, "BIGINT",
-                            Types.BIGINT, 22, 0, false));
-                }
-
-                public void doubleColumn(Column column)
-                {
-                    columns.add(new JdbcColumn(
-                            columnName, "DOUBLE PRECISION",
-                            Types.FLOAT, 24, 0, false));
-                }
-
-                public void stringColumn(Column column)
-                {
-                    columns.add(new JdbcColumn(
-                                columnName, "CLOB",
-                                Types.CLOB, 4000, 0, false));  // TODO size type param
-                }
-
-                public void timestampColumn(Column column)
-                {
-                    columns.add(new JdbcColumn(
-                                columnName, "TIMESTAMP",
-                                Types.TIMESTAMP, 26, 0, false));  // size type param is from postgresql.
-                }
-            });
-        }
-        return new JdbcSchema(columns.build());
-    }
-
-    public JdbcSchema newJdbcSchemaFromExistentTable(JdbcOutputConnection connection,
-            String tableName) throws SQLException
-    {
-        DatabaseMetaData dbm = connection.getMetaData();
-        String escape = dbm.getSearchStringEscape();
-        String schemaNamePattern = JdbcUtils.escapeSearchString(connection.getSchemaName(), escape);
-
-        ResultSet rs = dbm.getPrimaryKeys(null, schemaNamePattern, tableName);
-        ImmutableList.Builder<String> primaryKeysBuilder = ImmutableList.builder();
-        try {
-            while(rs.next()) {
-                primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
-            }
-        } finally {
-            rs.close();
-        }
-        ImmutableList<String> primaryKeys = primaryKeysBuilder.build();
-
-        String tableNamePattern = JdbcUtils.escapeSearchString(tableName, escape);
-        ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
-        rs = dbm.getColumns(null, schemaNamePattern, tableNamePattern, null);
-        try {
-            while(rs.next()) {
-                String columnName = rs.getString("COLUMN_NAME");
-                String typeName = rs.getString("TYPE_NAME");
-                boolean isPrimaryKey = primaryKeys.contains(columnName);
-                typeName = typeName.toUpperCase(Locale.ENGLISH);
-                int sqlType = rs.getInt("DATA_TYPE");
-                int colSize = rs.getInt("COLUMN_SIZE");
-                int decDigit = rs.getInt("DECIMAL_DIGITS");
-                if (rs.wasNull()) {
-                    decDigit = -1;
-                }
-                //rs.getString("IS_NULLABLE").equals("NO")  // "YES" or ""  // TODO
-                //rs.getString("COLUMN_DEF") // or null  // TODO
-                columns.add(new JdbcColumn(
-                            columnName, typeName,
-                            sqlType, colSize, decDigit, isPrimaryKey));
-            }
-        } finally {
-            rs.close();
-        }
-        return new JdbcSchema(columns.build());
-    }
-
-    private JdbcSchema matchSchemaByColumnNames(Schema inputSchema, JdbcSchema targetTableSchema)
-    {
-        ImmutableList.Builder<JdbcColumn> jdbcColumns = ImmutableList.builder();
-
-        outer : for (Column column : inputSchema.getColumns()) {
-            for (JdbcColumn jdbcColumn : targetTableSchema.getColumns()) {
-                if (jdbcColumn.getName().equals(column.getName())) {
-                    jdbcColumns.add(jdbcColumn);
-                    continue outer;
-                }
-            }
-
-            jdbcColumns.add(JdbcColumn.skipColumn());
-        }
-
-        return new JdbcSchema(jdbcColumns.build());
-    }
-
-    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final int taskIndex)
-    {
-        final PluginTask task = taskSource.loadTask(getTaskClass());
-        final Mode mode = task.getMode();
-
-        BatchInsert batch;
-        try {
-            batch = newBatchInsert(task);
-        } catch (IOException | SQLException ex) {
-            throw new RuntimeException(ex);
-        }
-        try {
-            PageReader reader = new PageReader(schema);
-            ColumnSetterFactory factory = newColumnSetterFactory(batch, reader, null);  // TODO TimestampFormatter
-
-            JdbcSchema loadSchema = task.getLoadSchema();
-
-            ImmutableList.Builder<JdbcColumn> insertColumns = ImmutableList.builder();
-            ImmutableList.Builder<ColumnSetter> columnSetters = ImmutableList.builder();
-            for (JdbcColumn c : loadSchema.getColumns()) {
-                if (c.isSkipColumn()) {
-                    columnSetters.add(factory.newSkipColumnSetter());
-                } else {
-                    columnSetters.add(factory.newColumnSetter(c));
-                    insertColumns.add(c);
-                }
-            }
-            final JdbcSchema insertSchema = new JdbcSchema(insertColumns.build());
-
-            final BatchInsert b = batch;
-            withRetry(new IdempotentSqlRunnable() {
-                public void run() throws SQLException
-                {
-                    String loadTable;
-                    boolean createTable;
-                    if (mode.usesMultipleLoadTables()) {
-                        // insert, truncate_insert, merge, replace
-                        loadTable = formatMultipleLoadTableName(task, taskIndex);
-                        JdbcOutputConnection con = newConnection(task, true, true);
-                        try {
-                            con.createTableIfNotExists(loadTable, insertSchema);
-                        } finally {
-                            con.close();
-                        }
-
-                    } else if (!mode.usesMultipleLoadTables() && mode.createAndSwapTable()) {
-                        // replace_inplace
-                        loadTable = task.getSwapTable().get();
-
-                    } else {
-                        // insert_direct
-                        loadTable = task.getTable();
-                    }
-                    b.prepare(loadTable, insertSchema);
-                }
-            });
-
-            PluginPageOutput output = newPluginPageOutput(reader, batch, columnSetters.build(), task);
-            batch = null;
-            return output;
-
-        } catch (SQLException | InterruptedException ex) {
-            throw new RuntimeException(ex);
-
-        } finally {
-            if (batch != null) {
-                try {
-                    batch.close();
-                } catch (IOException | SQLException ex) {
-                    throw new RuntimeException(ex);
-                }
-            }
-        }
-    }
-
-    protected ColumnSetterFactory newColumnSetterFactory(BatchInsert batch, PageReader pageReader,
-            TimestampFormatter timestampFormatter)
-    {
-    	return new ColumnSetterFactory(batch, pageReader, timestampFormatter);
-    }
-
-    protected PluginPageOutput newPluginPageOutput(PageReader reader,
-                                                   BatchInsert batch, List<ColumnSetter> columnSetters,
-                                                   PluginTask task)
-    {
-        return new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
-    }
-
-    public static class PluginPageOutput
-            implements TransactionalPageOutput
-    {
-        protected final List<Column> columns;
-        protected final List<ColumnSetter> columnSetters;
-        private final PageReader pageReader;
-        private final BatchInsert batch;
-        private final int batchSize;
-        private final int foraceBatchFlushSize;
-
-        public PluginPageOutput(PageReader pageReader,
-                BatchInsert batch, List<ColumnSetter> columnSetters,
-                int batchSize)
-        {
-            this.pageReader = pageReader;
-            this.batch = batch;
-            this.columns = pageReader.getSchema().getColumns();
-            this.columnSetters = columnSetters;
-            this.batchSize = batchSize;
-            this.foraceBatchFlushSize = batchSize * 2;
-        }
-
-        @Override
-        public void add(Page page)
-        {
-            try {
-                pageReader.setPage(page);
-                while (pageReader.nextRecord()) {
-                    if (batch.getBatchWeight() > foraceBatchFlushSize) {
-                        batch.flush();
-                    }
-                    handleColumnsSetters();
-                    batch.add();
-                }
-                if (batch.getBatchWeight() > batchSize) {
-                    batch.flush();
-                }
-            } catch (IOException | SQLException ex) {
-                throw new RuntimeException(ex);
-            }
-        }
-
-        @Override
-        public void finish()
-        {
-            try {
-                batch.finish();
-            } catch (IOException | SQLException ex) {
-                throw new RuntimeException(ex);
-            }
-        }
-
-        @Override
-        public void close()
-        {
-            try {
-                batch.close();
-            } catch (IOException | SQLException ex) {
-                throw new RuntimeException(ex);
-            }
-        }
-
-        @Override
-        public void abort()
-        {
-        }
-
-        @Override
-        public CommitReport commit()
-        {
-            return Exec.newCommitReport();
-        }
-
-        protected void handleColumnsSetters()
-        {
-            int size = columnSetters.size();
-            for (int i=0; i < size; i++) {
-                columns.get(i).visit(columnSetters.get(i));
-            }
-        }
-
-    }
-
-    public static interface IdempotentSqlRunnable
-    {
-        public void run() throws SQLException;
-    }
-
-    protected void withRetry(IdempotentSqlRunnable op)
-            throws SQLException, InterruptedException
-    {
-        withRetry(op, "Operation failed");
-    }
-
-    protected void withRetry(final IdempotentSqlRunnable op, final String errorMessage)
-            throws SQLException, InterruptedException
-    {
-        try {
-            retryExecutor()
-                .setRetryLimit(12)
-                .setInitialRetryWait(1000)
-                .setMaxRetryWait(30 * 60 * 1000)
-                .runInterruptible(new IdempotentOperation<Void>() {
-                    public Void call() throws Exception
-                    {
-                        op.run();
-                        return null;
-                    }
-
-                    public void onRetry(Throwable exception, int retryCount, int retryLimit, int retryWait)
-                    {
-                        if (exception instanceof SQLException) {
-                            SQLException ex = (SQLException) exception;
-                            String sqlState = ex.getSQLState();
-                            int errorCode = ex.getErrorCode();
-                            logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
-                                    errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
-                                    buildExceptionMessage(exception));
-                        } else {
-                            logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
-                                    errorMessage, retryCount, retryLimit, retryWait/1000,
-                                    buildExceptionMessage(exception));
-                        }
-                        if (retryCount % 3 == 0) {
-                            logger.info("Error details:", exception);
-                        }
-                    }
-
-                    public void onGiveup(Throwable firstException, Throwable lastException)
-                    {
-                        if (firstException instanceof SQLException) {
-                            SQLException ex = (SQLException) firstException;
-                            String sqlState = ex.getSQLState();
-                            int errorCode = ex.getErrorCode();
-                            logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
-                        }
-                    }
-
-                    public boolean isRetryableException(Throwable exception)
-                    {
-                        //if (exception instanceof SQLException) {
-                        //    SQLException ex = (SQLException) exception;
-                        //    String sqlState = ex.getSQLState();
-                        //    int errorCode = ex.getErrorCode();
-                        //    return isRetryableSQLException(ex);
-                        //}
-                        return false;  // TODO
-                    }
-                });
-
-        } catch (ExecutionException ex) {
-            Throwable cause = ex.getCause();
-            Throwables.propagateIfInstanceOf(cause, SQLException.class);
-            throw Throwables.propagate(cause);
-        }
-    }
-
-    private String buildExceptionMessage(Throwable ex) {
-        StringBuilder sb = new StringBuilder();
-        sb.append(ex.getMessage());
-        if (ex.getCause() != null) {
-            buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
-        }
-        return sb.toString();
-    }
-
-    private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
-        if (!lastMessage.equals(ex.getMessage())) {
-            // suppress same messages
-            sb.append(" < ");
-            sb.append(ex.getMessage());
-        }
-        if (ex.getCause() == null) {
-            return;
-        }
-        buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
-    }
-}
+package org.embulk.output.jdbc;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Locale;
+import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.sql.Types;
+import java.sql.ResultSet;
+import java.sql.DatabaseMetaData;
+import java.sql.SQLException;
+import org.slf4j.Logger;
+import org.joda.time.DateTimeZone;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonValue;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.google.common.base.Optional;
+import com.google.common.base.Function;
+import com.google.common.base.Supplier;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.embulk.config.CommitReport;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigException;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.Column;
+import org.embulk.spi.ColumnVisitor;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.PluginClassLoader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.type.StringType;
+import org.embulk.spi.time.Timestamp;
+import org.embulk.spi.time.TimestampFormat;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.output.jdbc.setter.ColumnSetter;
+import org.embulk.output.jdbc.setter.ColumnSetterFactory;
+import org.embulk.output.jdbc.setter.ColumnSetterVisitor;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+import static org.embulk.output.jdbc.JdbcSchema.filterSkipColumns;
+
+public abstract class AbstractJdbcOutputPlugin
+        implements OutputPlugin
+{
+    private final static Set<String> loadedJarGlobs = new HashSet<String>();
+
+    protected final Logger logger = Exec.getLogger(getClass());
+
+    public interface PluginTask
+            extends Task
+    {
+        @Config("options")
+        @ConfigDefault("{}")
+        public Properties getOptions();
+
+        @Config("table")
+        public String getTable();
+
+        @Config("mode")
+        public Mode getMode();
+
+        @Config("batch_size")
+        @ConfigDefault("16777216")
+        // TODO set minimum number
+        public int getBatchSize();
+
+        @Config("merge_keys")
+        @ConfigDefault("null")
+        public Optional<List<String>> getMergeKeys();
+
+        @Config("column_options")
+        @ConfigDefault("{}")
+        public Map<String, JdbcColumnOption> getColumnOptions();
+
+        @Config("default_timezone")
+        @ConfigDefault("\"UTC\"")
+        public DateTimeZone getDefaultTimeZone();
+
+        public void setMergeKeys(Optional<List<String>> keys);
+
+        public void setFeatures(Features features);
+        public Features getFeatures();
+
+        public JdbcSchema getTargetTableSchema();
+        public void setTargetTableSchema(JdbcSchema schema);
+
+        public Optional<List<String>> getIntermediateTables();
+        public void setIntermediateTables(Optional<List<String>> names);
+    }
+
+    public static class Features
+    {
+        private int maxTableNameLength = 64;
+        private Set<Mode> supportedModes = ImmutableSet.copyOf(Mode.values());
+        private boolean ignoreMergeKeys = false;
+
+        public Features()
+        { }
+
+        @JsonProperty
+        public int getMaxTableNameLength()
+        {
+            return maxTableNameLength;
+        }
+
+        @JsonProperty
+        public Features setMaxTableNameLength(int bytes)
+        {
+            this.maxTableNameLength = bytes;
+            return this;
+        }
+
+        @JsonProperty
+        public Set<Mode> getSupportedModes()
+        {
+            return supportedModes;
+        }
+
+        @JsonProperty
+        public Features setSupportedModes(Set<Mode> modes)
+        {
+            this.supportedModes = modes;
+            return this;
+        }
+
+        @JsonProperty
+        public boolean getIgnoreMergeKeys()
+        {
+            return ignoreMergeKeys;
+        }
+
+        @JsonProperty
+        public Features setIgnoreMergeKeys(boolean value)
+        {
+            this.ignoreMergeKeys = value;
+            return this;
+        }
+    }
+
+    protected void loadDriverJar(String glob)
+    {
+        synchronized (loadedJarGlobs) {
+            if (!loadedJarGlobs.contains(glob)) {
+                // TODO match glob
+                PluginClassLoader loader = (PluginClassLoader) getClass().getClassLoader();
+                loader.addPath(Paths.get(glob));
+                loadedJarGlobs.add(glob);
+            }
+        }
+    }
+
+    // for subclasses to add @Config
+    protected Class<? extends PluginTask> getTaskClass()
+    {
+        return PluginTask.class;
+    }
+
+    protected abstract Features getFeatures(PluginTask task);
+
+    protected abstract JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation);
+
+    protected abstract BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException;
+
+    protected JdbcOutputConnection newConnection(PluginTask task, boolean retryableMetadataOperation,
+            boolean autoCommit) throws SQLException
+    {
+        return getConnector(task, retryableMetadataOperation).connect(autoCommit);
+    }
+
+    public enum Mode {
+        INSERT,
+        INSERT_DIRECT,
+        MERGE,
+        MERGE_DIRECT,
+        TRUNCATE_INSERT,
+        REPLACE;
+
+        @JsonValue
+        @Override
+        public String toString()
+        {
+            return name().toLowerCase(Locale.ENGLISH);
+        }
+
+        @JsonCreator
+        public static Mode fromString(String value)
+        {
+            switch(value) {
+            case "insert":
+                return INSERT;
+            case "insert_direct":
+                return INSERT_DIRECT;
+            case "merge":
+                return MERGE;
+            case "merge_direct":
+                return MERGE_DIRECT;
+            case "truncate_insert":
+                return TRUNCATE_INSERT;
+            case "replace":
+                return REPLACE;
+            default:
+                throw new ConfigException(String.format("Unknown mode '%s'. Supported modes are insert, insert_direct, merge, merge_direct, truncate_insert, replace", value));
+            }
+        }
+
+        /**
+         * True if this mode directly modifies the target table without creating intermediate tables.
+         */
+        public boolean isDirectModify()
+        {
+            return this == INSERT_DIRECT || this == MERGE_DIRECT;
+        }
+
+        /**
+         * True if this mode merges records on unique keys
+         */
+        public boolean isMerge()
+        {
+            return this == MERGE || this == MERGE_DIRECT;
+        }
+
+        /**
+         * True if this mode creates intermediate table for each tasks.
+         */
+        public boolean tempTablePerTask()
+        {
+            return this == INSERT || this == MERGE || this == TRUNCATE_INSERT /*this == REPLACE_VIEW*/;
+        }
+
+        /**
+         * True if this mode truncates the target table before committing intermediate tables
+         */
+        public boolean truncateBeforeCommit()
+        {
+            return this == TRUNCATE_INSERT;
+        }
+
+        /**
+         * True if this mode uses MERGE statement to commit intermediate tables to the target table
+         */
+        public boolean commitByMerge()
+        {
+            return this == MERGE;
+        }
+
+        /**
+         * True if this mode overwrites schema of the target tables
+         */
+        public boolean ignoreTargetTableSchema()
+        {
+            return this == REPLACE /*|| this == REPLACE_VIEW*/;
+        }
+
+        /**
+         * True if this mode swaps the target tables with intermediate tables to commit
+         */
+        public boolean commitBySwapTable()
+        {
+            return this == REPLACE;
+        }
+    }
+
+    public ConfigDiff transaction(ConfigSource config,
+            Schema schema, int taskCount,
+            OutputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(getTaskClass());
+        Features features = getFeatures(task);
+        task.setFeatures(features);
+
+        if (!features.getSupportedModes().contains(task.getMode())) {
+            throw new ConfigException(String.format("This output type doesn't support '%s'. Supported modes are: %s", task.getMode(), features.getSupportedModes()));
+        }
+
+        task = begin(task, schema, taskCount);
+        control.run(task.dump());
+        return commit(task, schema, taskCount);
+    }
+
+    public ConfigDiff resume(TaskSource taskSource,
+            Schema schema, int taskCount,
+            OutputPlugin.Control control)
+    {
+        PluginTask task = taskSource.loadTask(getTaskClass());
+
+        if (!task.getMode().tempTablePerTask()) {
+            throw new UnsupportedOperationException("inplace mode is not resumable. You need to delete partially-loaded records from the database and restart the entire transaction.");
+        }
+
+        task = begin(task, schema, taskCount);
+        control.run(task.dump());
+        return commit(task, schema, taskCount);
+    }
+
+    protected String getTransactionUniqueName()
+    {
+        // TODO use uuid?
+        Timestamp t = Exec.session().getTransactionTime();
+        return String.format("%016x%08x", t.getEpochSecond(), t.getNano());
+    }
+
+    private PluginTask begin(final PluginTask task,
+            final Schema schema, final int taskCount)
+    {
+        try {
+            withRetry(new IdempotentSqlRunnable() {  // no intermediate data if isDirectModify == true
+                public void run() throws SQLException
+                {
+                    JdbcOutputConnection con = newConnection(task, true, false);
+                    try {
+                        doBegin(con, task, schema, taskCount);
+                    } finally {
+                        con.close();
+                    }
+                }
+            });
+        } catch (SQLException | InterruptedException ex) {
+            throw new RuntimeException(ex);
+        }
+        return task;
+    }
+
+    private ConfigDiff commit(final PluginTask task,
+            Schema schema, final int taskCount)
+    {
+        if (!task.getMode().isDirectModify()) {  // no intermediate data if isDirectModify == true
+            try {
+                withRetry(new IdempotentSqlRunnable() {
+                    public void run() throws SQLException
+                    {
+                        JdbcOutputConnection con = newConnection(task, false, false);
+                        try {
+                            doCommit(con, task, taskCount);
+                        } finally {
+                            con.close();
+                        }
+                    }
+                });
+            } catch (SQLException | InterruptedException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+        return Exec.newConfigDiff();
+    }
+
+    public void cleanup(TaskSource taskSource,
+            Schema schema, final int taskCount,
+            final List<CommitReport> successCommitReports)
+    {
+        final PluginTask task = taskSource.loadTask(getTaskClass());
+
+        if (!task.getMode().isDirectModify()) {  // no intermediate data if isDirectModify == true
+            try {
+                withRetry(new IdempotentSqlRunnable() {
+                    public void run() throws SQLException
+                    {
+                        JdbcOutputConnection con = newConnection(task, true, true);
+                        try {
+                            doCleanup(con, task, taskCount, successCommitReports);
+                        } finally {
+                            con.close();
+                        }
+                    }
+                });
+            } catch (SQLException | InterruptedException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+
+    protected void doBegin(JdbcOutputConnection con,
+            PluginTask task, final Schema schema, int taskCount) throws SQLException
+    {
+        if (schema.getColumnCount() == 0) {
+            throw new ConfigException("task count == 0 is not supported");
+        }
+
+        Mode mode = task.getMode();
+        logger.info("Using {} mode", mode);
+
+        Optional<JdbcSchema> initialTargetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getTable());
+
+        // TODO get CREATE TABLE statement from task if set
+        JdbcSchema newTableSchema = applyColumnOptionsToNewTableSchema(
+                initialTargetTableSchema.or(new Supplier<JdbcSchema>() {
+                    public JdbcSchema get()
+                    {
+                        return newJdbcSchemaForNewTable(schema);
+                    }
+                }),
+                task.getColumnOptions());
+
+        // create intermediate tables
+        if (!mode.isDirectModify()) {
+            // direct modify mode doesn't need intermediate tables.
+            ImmutableList.Builder<String> intermTableNames = ImmutableList.builder();
+            if (mode.tempTablePerTask()) {
+                String namePrefix = generateIntermediateTableNamePrefix(task.getTable(), con, 3, task.getFeatures().getMaxTableNameLength());
+                for (int i=0; i < taskCount; i++) {
+                    intermTableNames.add(namePrefix + String.format("%03d", i));
+                }
+            } else {
+                String name = generateIntermediateTableNamePrefix(task.getTable(), con, 0, task.getFeatures().getMaxTableNameLength());
+                intermTableNames.add(name);
+            }
+            // create the intermediate tables here
+            task.setIntermediateTables(Optional.<List<String>>of(intermTableNames.build()));
+            for (String name : task.getIntermediateTables().get()) {
+                // DROP TABLE IF EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
+                con.dropTableIfExists(name);
+                // CREATE TABLE IF NOT EXISTS xyz__0000000054d92dee1e452158_bulk_load_temp
+                con.createTableIfNotExists(name, newTableSchema);
+            }
+        } else {
+            task.setIntermediateTables(Optional.<List<String>>absent());
+        }
+
+        // build JdbcSchema from a table
+        JdbcSchema targetTableSchema;
+        if (mode.ignoreTargetTableSchema() && taskCount != 0) {
+            String firstItermTable = task.getIntermediateTables().get().get(0);
+            targetTableSchema = newJdbcSchemaFromTableIfExists(con, firstItermTable).get();
+        } else if (initialTargetTableSchema.isPresent()) {
+            targetTableSchema = initialTargetTableSchema.get();
+        } else {
+            // also create the target table if not exists
+            // CREATE TABLE IF NOT EXISTS xyz
+            con.createTableIfNotExists(task.getTable(), newTableSchema);
+            targetTableSchema = newJdbcSchemaFromTableIfExists(con, task.getTable()).get();
+        }
+        task.setTargetTableSchema(matchSchemaByColumnNames(schema, targetTableSchema));
+
+        // validate column_options
+        newColumnSetters(
+                new ColumnSetterFactory(null, task.getDefaultTimeZone()),  // TODO create a dummy BatchInsert
+                task.getTargetTableSchema(), schema,
+                task.getColumnOptions());
+
+        // normalize merge_key parameter for merge modes
+        if (mode.isMerge()) {
+            Optional<List<String>> mergeKeys = task.getMergeKeys();
+            if (task.getFeatures().getIgnoreMergeKeys()) {
+                if (mergeKeys.isPresent()) {
+                    throw new ConfigException("This output type does not accept 'merge_key' option.");
+                }
+                task.setMergeKeys(Optional.<List<String>>of(ImmutableList.<String>of()));
+            } else if (mergeKeys.isPresent()) {
+                if (task.getMergeKeys().get().isEmpty()) {
+                    throw new ConfigException("Empty 'merge_keys' option is invalid.");
+                }
+                for (String key : mergeKeys.get()) {
+                    if (!targetTableSchema.findColumn(key).isPresent()) {
+                        throw new ConfigException(String.format("Merge key '%s' does not exist in the target table.", key));
+                    }
+                }
+            } else {
+                ImmutableList.Builder<String> builder = ImmutableList.builder();
+                for (JdbcColumn column : targetTableSchema.getColumns()) {
+                    if (column.isUniqueKey()) {
+                        builder.add(column.getName());
+                    }
+                }
+                task.setMergeKeys(Optional.<List<String>>of(builder.build()));
+                if (task.getMergeKeys().get().isEmpty()) {
+                    throw new ConfigException("Merging mode is used but the target table does not have primary keys. Please set merge_keys option.");
+                }
+            }
+            logger.info("Using merge keys: {}", task.getMergeKeys().get());
+        } else {
+            task.setMergeKeys(Optional.<List<String>>absent());
+        }
+    }
+
+    protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con, int suffixLength, int maxLength) throws SQLException
+    {
+        Charset tableNameCharset = con.getTableNameCharset();
+        String tableName = baseTableName;
+        String suffix = "_bl_tmp";
+        String uniqueSuffix = getTransactionUniqueName() + suffix;
+
+        // way to count length of table name varies by DBMSs (bytes or characters),
+        // so truncate swap table name by one character.
+        while (!checkTableNameLength(tableName + "_" + uniqueSuffix, tableNameCharset, suffixLength, maxLength)) {
+            if (uniqueSuffix.length() > 8 + suffix.length()) {
+                // truncate transaction unique name
+                // (include 8 characters of the transaction name at least)
+                uniqueSuffix = uniqueSuffix.substring(1);
+            } else {
+                if (tableName.isEmpty()) {
+                    throw new ConfigException("Table name is too long to generate temporary table name");
+                }
+                // truncate table name
+                tableName = tableName.substring(0, tableName.length() - 1);
+                //if (!connection.tableExists(tableName)) {
+                // TODO this doesn't help. Rather than truncating more characters,
+                //      here needs to replace characters with random characters. But
+                //      to make the result deterministic. So, an idea is replacing
+                //      the last character to the first (second, third, ... for each loop)
+                //      of md5(original table name).
+                //}
+            }
+
+        }
+        return tableName + "_" + uniqueSuffix;
+    }
+
+    private static JdbcSchema applyColumnOptionsToNewTableSchema(JdbcSchema schema, final Map<String, JdbcColumnOption> columnOptions)
+    {
+        return new JdbcSchema(Lists.transform(schema.getColumns(), new Function<JdbcColumn, JdbcColumn>() {
+            public JdbcColumn apply(JdbcColumn c)
+            {
+                JdbcColumnOption option = columnOptionOf(columnOptions, c);
+                if (option.getType().isPresent()) {
+                    return JdbcColumn.newTypeDeclaredColumn(
+                            c.getName(), Types.OTHER,  // sqlType, isNotNull, and isUniqueKey are ignored
+                            option.getType().get(), false, false);
+                }
+                return c;
+            }
+        }));
+    }
+
+    protected static List<ColumnSetter> newColumnSetters(ColumnSetterFactory factory,
+            JdbcSchema targetTableSchema, Schema inputValueSchema,
+            Map<String, JdbcColumnOption> columnOptions)
+    {
+        ImmutableList.Builder<ColumnSetter> builder = ImmutableList.builder();
+        int schemaColumnIndex = 0;
+        for (JdbcColumn targetColumn : targetTableSchema.getColumns()) {
+            if (targetColumn.isSkipColumn()) {
+                builder.add(factory.newSkipColumnSetter());
+            } else {
+                //String columnOptionKey = inputValueSchema.getColumn(schemaColumnIndex).getName();
+                JdbcColumnOption option = columnOptionOf(columnOptions, targetColumn);
+                builder.add(factory.newColumnSetter(targetColumn, option));
+                schemaColumnIndex++;
+            }
+        }
+        return builder.build();
+    }
+
+    private static JdbcColumnOption columnOptionOf(Map<String, JdbcColumnOption> columnOptions, JdbcColumn targetColumn)
+    {
+        return Optional.fromNullable(columnOptions.get(targetColumn.getName())).or(
+                    // default column option
+                    new Supplier<JdbcColumnOption>()
+                    {
+                        public JdbcColumnOption get()
+                        {
+                            return Exec.newConfigSource().loadConfig(JdbcColumnOption.class);
+                        }
+                    });
+    }
+
+    private boolean checkTableNameLength(String tableName, Charset tableNameCharset, int suffixLength, int maxLength)
+    {
+        return tableNameCharset.encode(tableName).remaining() + suffixLength <= maxLength;
+    }
+
+    protected void doCommit(JdbcOutputConnection con, PluginTask task, int taskCount)
+        throws SQLException
+    {
+        JdbcSchema schema = filterSkipColumns(task.getTargetTableSchema());
+
+        switch (task.getMode()) {
+        case INSERT_DIRECT:
+        case MERGE_DIRECT:
+            // already done
+            break;
+
+        case INSERT:
+            // aggregate insert into target
+            con.collectInsert(task.getIntermediateTables().get(), schema, task.getTable(), false);
+            break;
+
+        case TRUNCATE_INSERT:
+            // truncate & aggregate insert into target
+            con.collectInsert(task.getIntermediateTables().get(), schema, task.getTable(), true);
+            break;
+
+        case MERGE:
+            // aggregate merge into target
+            con.collectMerge(task.getIntermediateTables().get(), schema, task.getTable(), task.getMergeKeys().get());
+            break;
+
+        case REPLACE:
+            // swap table
+            con.replaceTable(task.getIntermediateTables().get().get(0), schema, task.getTable());
+            break;
+        }
+    }
+
+    protected void doCleanup(JdbcOutputConnection con, PluginTask task, int taskCount,
+            List<CommitReport> successCommitReports)
+        throws SQLException
+    {
+        if (task.getIntermediateTables().isPresent()) {
+            for (String intermTable : task.getIntermediateTables().get()) {
+                con.dropTableIfExists(intermTable);
+            }
+        }
+    }
+
+    protected JdbcSchema newJdbcSchemaForNewTable(Schema schema)
+    {
+        final ImmutableList.Builder<JdbcColumn> columns = ImmutableList.builder();
+        for (Column c : schema.getColumns()) {
+            final String columnName = c.getName();
+            c.visit(new ColumnVisitor() {
+                public void booleanColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.BOOLEAN, "BOOLEAN",
+                            1, 0, false, false));
+                }
+
+                public void longColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.BIGINT, "BIGINT",
+                            22, 0, false, false));
+                }
+
+                public void doubleColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.FLOAT, "DOUBLE PRECISION",
+                            24, 0, false, false));
+                }
+
+                public void stringColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.CLOB, "CLOB",
+                            4000, 0, false, false));  // TODO size type param
+                }
+
+                public void timestampColumn(Column column)
+                {
+                    columns.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, Types.TIMESTAMP, "TIMESTAMP",
+                            26, 0, false, false));  // size type param is from postgresql
+                }
+            });
+        }
+        return new JdbcSchema(columns.build());
+    }
+
+    public Optional<JdbcSchema> newJdbcSchemaFromTableIfExists(JdbcOutputConnection connection,
+            String tableName) throws SQLException
+    {
+        DatabaseMetaData dbm = connection.getMetaData();
+        String escape = dbm.getSearchStringEscape();
+
+        ResultSet rs = dbm.getPrimaryKeys(null, connection.getSchemaName(), tableName);
+        ImmutableSet.Builder<String> primaryKeysBuilder = ImmutableSet.builder();
+        try {
+            while(rs.next()) {
+                primaryKeysBuilder.add(rs.getString("COLUMN_NAME"));
+            }
+        } finally {
+            rs.close();
+        }
+        ImmutableSet<String> primaryKeys = primaryKeysBuilder.build();
+
+        ImmutableList.Builder<JdbcColumn> builder = ImmutableList.builder();
+        rs = dbm.getColumns(null,
+                JdbcUtils.escapeSearchString(connection.getSchemaName(), escape),
+                JdbcUtils.escapeSearchString(tableName, escape),
+                null);
+        try {
+            while (rs.next()) {
+                String columnName = rs.getString("COLUMN_NAME");
+                String simpleTypeName = rs.getString("TYPE_NAME").toUpperCase(Locale.ENGLISH);
+                boolean isUniqueKey = primaryKeys.contains(columnName);
+                int sqlType = rs.getInt("DATA_TYPE");
+                int colSize = rs.getInt("COLUMN_SIZE");
+                int decDigit = rs.getInt("DECIMAL_DIGITS");
+                if (rs.wasNull()) {
+                    decDigit = -1;
+                }
+                boolean isNotNull = "NO".equals(rs.getString("IS_NULLABLE"));
+                //rs.getString("COLUMN_DEF") // or null  // TODO
+                builder.add(JdbcColumn.newGenericTypeColumn(
+                            columnName, sqlType, simpleTypeName, colSize, decDigit, isNotNull, isUniqueKey));
+                // We can't get declared column name using JDBC API.
+                // Subclasses need to overwrite it.
+            }
+        } finally {
+            rs.close();
+        }
+        List<JdbcColumn> columns = builder.build();
+        if (columns.isEmpty()) {
+            return Optional.absent();
+        } else {
+            return Optional.of(new JdbcSchema(columns));
+        }
+    }
+
+    private JdbcSchema matchSchemaByColumnNames(Schema inputSchema, JdbcSchema targetTableSchema)
+    {
+        ImmutableList.Builder<JdbcColumn> jdbcColumns = ImmutableList.builder();
+
+        for (Column column : inputSchema.getColumns()) {
+            Optional<JdbcColumn> c = targetTableSchema.findColumn(column.getName());
+            jdbcColumns.add(c.or(JdbcColumn.skipColumn()));
+        }
+
+        return new JdbcSchema(jdbcColumns.build());
+    }
+
+    public TransactionalPageOutput open(TaskSource taskSource, Schema schema, final int taskIndex)
+    {
+        final PluginTask task = taskSource.loadTask(getTaskClass());
+        final Mode mode = task.getMode();
+
+        // instantiate BatchInsert without table name
+        BatchInsert batch = null;
+        try {
+            batch = newBatchInsert(task,
+                    task.getMode() == Mode.MERGE_DIRECT ?
+                        task.getMergeKeys() :
+                        Optional.<List<String>>absent());
+        } catch (IOException | SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+
+        try {
+            // configure PageReader -> BatchInsert
+            PageReader reader = new PageReader(schema);
+
+            List<ColumnSetter> columnSetters = newColumnSetters(
+                    new ColumnSetterFactory(batch, task.getDefaultTimeZone()),
+                    task.getTargetTableSchema(), schema,
+                    task.getColumnOptions());
+            JdbcSchema insertIntoSchema = filterSkipColumns(task.getTargetTableSchema());
+
+            // configure BatchInsert -> an intermediate table (!isDirectModify) or the target table (isDirectModify)
+            String destTable;
+            if (mode.tempTablePerTask()) {
+                destTable = task.getIntermediateTables().get().get(taskIndex);
+            } else if (mode.isDirectModify()) {
+                destTable = task.getTable();
+            } else {
+                destTable = task.getIntermediateTables().get().get(0);
+            }
+            batch.prepare(destTable, insertIntoSchema);
+
+            PluginPageOutput output = new PluginPageOutput(reader, batch, columnSetters, task.getBatchSize());
+            batch = null;
+            return output;
+
+        } catch (SQLException ex) {
+            throw new RuntimeException(ex);
+
+        } finally {
+            if (batch != null) {
+                try {
+                    batch.close();
+                } catch (IOException | SQLException ex) {
+                    throw new RuntimeException(ex);
+                }
+            }
+        }
+    }
+
+    public static class PluginPageOutput
+            implements TransactionalPageOutput
+    {
+        protected final List<Column> columns;
+        protected final List<ColumnSetterVisitor> columnVisitors;
+        private final PageReader pageReader;
+        private final BatchInsert batch;
+        private final int batchSize;
+        private final int forceBatchFlushSize;
+
+        public PluginPageOutput(final PageReader pageReader,
+                BatchInsert batch, List<ColumnSetter> columnSetters,
+                int batchSize)
+        {
+            this.pageReader = pageReader;
+            this.batch = batch;
+            this.columns = pageReader.getSchema().getColumns();
+            this.columnVisitors = ImmutableList.copyOf(Lists.transform(
+                        columnSetters, new Function<ColumnSetter, ColumnSetterVisitor>() {
+                            public ColumnSetterVisitor apply(ColumnSetter setter)
+                            {
+                                return new ColumnSetterVisitor(pageReader, setter);
+                            }
+                        }));
+            this.batchSize = batchSize;
+            this.forceBatchFlushSize = batchSize * 2;
+        }
+
+        @Override
+        public void add(Page page)
+        {
+            try {
+                pageReader.setPage(page);
+                while (pageReader.nextRecord()) {
+                    if (batch.getBatchWeight() > forceBatchFlushSize) {
+                        batch.flush();
+                    }
+                    handleColumnsSetters();
+                    batch.add();
+                }
+                if (batch.getBatchWeight() > batchSize) {
+                    batch.flush();
+                }
+            } catch (IOException | SQLException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public void finish()
+        {
+            try {
+                batch.finish();
+            } catch (IOException | SQLException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public void close()
+        {
+            try {
+                batch.close();
+            } catch (IOException | SQLException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+
+        @Override
+        public void abort()
+        {
+        }
+
+        @Override
+        public CommitReport commit()
+        {
+            return Exec.newCommitReport();
+        }
+
+        protected void handleColumnsSetters()
+        {
+            int size = columnVisitors.size();
+            for (int i=0; i < size; i++) {
+                columns.get(i).visit(columnVisitors.get(i));
+            }
+        }
+    }
+
+    public static interface IdempotentSqlRunnable
+    {
+        public void run() throws SQLException;
+    }
+
+    protected void withRetry(IdempotentSqlRunnable op)
+            throws SQLException, InterruptedException
+    {
+        withRetry(op, "Operation failed");
+    }
+
+    protected void withRetry(final IdempotentSqlRunnable op, final String errorMessage)
+            throws SQLException, InterruptedException
+    {
+        try {
+            retryExecutor()
+                .withRetryLimit(12)
+                .withInitialRetryWait(1000)
+                .withMaxRetryWait(30 * 60 * 1000)
+                .runInterruptible(new Retryable<Void>() {
+                    public Void call() throws Exception
+                    {
+                        op.run();
+                        return null;
+                    }
+
+                    public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                    {
+                        if (exception instanceof SQLException) {
+                            SQLException ex = (SQLException) exception;
+                            String sqlState = ex.getSQLState();
+                            int errorCode = ex.getErrorCode();
+                            logger.warn("{} ({}:{}), retrying {}/{} after {} seconds. Message: {}",
+                                    errorMessage, errorCode, sqlState, retryCount, retryLimit, retryWait/1000,
+                                    buildExceptionMessage(exception));
+                        } else {
+                            logger.warn("{}, retrying {}/{} after {} seconds. Message: {}",
+                                    errorMessage, retryCount, retryLimit, retryWait/1000,
+                                    buildExceptionMessage(exception));
+                        }
+                        if (retryCount % 3 == 0) {
+                            logger.info("Error details:", exception);
+                        }
+                    }
+
+                    public void onGiveup(Exception firstException, Exception lastException)
+                    {
+                        if (firstException instanceof SQLException) {
+                            SQLException ex = (SQLException) firstException;
+                            String sqlState = ex.getSQLState();
+                            int errorCode = ex.getErrorCode();
+                            logger.error("{} ({}:{})", errorMessage, errorCode, sqlState);
+                        }
+                    }
+
+                    public boolean isRetryableException(Exception exception)
+                    {
+                        //if (exception instanceof SQLException) {
+                        //    SQLException ex = (SQLException) exception;
+                        //    String sqlState = ex.getSQLState();
+                        //    int errorCode = ex.getErrorCode();
+                        //    return isRetryableSQLException(ex);
+                        //}
+                        return false;  // TODO
+                    }
+                });
+
+        } catch (ExecutionException ex) {
+            Throwable cause = ex.getCause();
+            Throwables.propagateIfInstanceOf(cause, SQLException.class);
+            throw Throwables.propagate(cause);
+        }
+    }
+
+    private String buildExceptionMessage(Throwable ex) {
+        StringBuilder sb = new StringBuilder();
+        sb.append(ex.getMessage());
+        if (ex.getCause() != null) {
+            buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
+        }
+        return sb.toString();
+    }
+
+    private void buildExceptionMessageCont(StringBuilder sb, Throwable ex, String lastMessage) {
+        if (!lastMessage.equals(ex.getMessage())) {
+            // suppress same messages
+            sb.append(" < ");
+            sb.append(ex.getMessage());
+        }
+        if (ex.getCause() == null) {
+            return;
+        }
+        buildExceptionMessageCont(sb, ex.getCause(), ex.getMessage());
+    }
+}