embulk-output-mysql 0.7.7 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf58d4882f474c11125f531945a5048f350c0f39
-  data.tar.gz: 2f28afb1f6bf1bcfdc700595f59eaaf0143984c7
+  metadata.gz: 6da5ba42634421a68e79de5f9c753a76d8406786
+  data.tar.gz: 7f80247c0f4c5fa8b973d9dca3914151f90815e5
 SHA512:
-  metadata.gz: 227b8b0b5a95eb0c464a98a67a2824fb79d93c0491b97bcb729c0b15f624307d86095cdd5c1ed74afc2a424215ca87df070046fdcb2990f83b9398ac72e4e671
-  data.tar.gz: 9536f2f57ae9a11a48a0b594b4f67157a8ff35bb83ba3528db9cf05544fec85c60c4ae8a0fd17b979d92e35c5b413e058c41236e0751ad6786fda2e821519fcd
+  metadata.gz: be36e44971e9d805842958a0acd90f69cbde13c141df4e727aff2fcd05158452d463ebfe7d36ff476643e76a71f0793494dc694e1687083ea46892ef5e59955f
+  data.tar.gz: 2bbf372969dd29e6368d32da9c29db14322f9035c4291696853d7c8d6f7a75261dae5086b063a09efe5435fa53a57359eb00d8f22b127ec9d493affd86600d46

data/README.md

@@ -10,12 +10,17 @@ MySQL output plugin for Embulk loads records to MySQL.
 
 ## Configuration
 
+- **driver_path**: path to the jar file of the MySQL JDBC driver. If not set, the bundled JDBC driver (MySQL Connector/J 5.1.44) will be used (string). NOTE: the bundled JDBC driver version was upgrade from 5.1.34 to 5.1.44 at embulk-output-mysql 0.8.0 . Please check release notes for MySQL Connector/J (https://dev.mysql.com/doc/relnotes/connector-j/5.1/en/news-5-1.html) .
 - **host**: database host name (string, required)
 - **port**: database port number (integer, default: 3306)
 - **user**: database login user name (string, required)
 - **password**: database login password (string, default: "")
+- **ssl**: use SSL to connect to the database (string, default: `disable`. `enable` uses SSL without server-side validation and `verify` checks the certificate. For compatibility reasons, `true` behaves as `enable` and `false` behaves as `disable`.)
 - **database**: destination database name (string, required)
+- **temp_database**: database name for intermediate tables. by default, intermediate tables will be created in the database specified by `database`. (string, optional)
 - **table**: destination table name (string, required)
+- **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **options**: extra connection properties (hash, default: {})
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
@@ -118,3 +123,19 @@ out:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+You need to create 'mysql.yml' as follows.
+```
+type: mysql
+host: localhost
+port: 3306
+database: database
+user: user
+password: pass
+```
+
+```
+$ EMBULK_OUTPUT_MYSQL_TEST_CONFIG=mysql.yml ./gradlew :embulk-output-mysql:check --info
+```

data/build.gradle

@@ -1,7 +1,7 @@
 dependencies {
     compile project(':embulk-output-jdbc')
-    compile 'mysql:mysql-connector-java:5.1.34'
+    compile 'mysql:mysql-connector-java:5.1.44'
+    defaultJdbcDriver 'mysql:mysql-connector-java:5.1.44'
 
-    testCompile 'org.embulk:embulk-standards:0.8.8'
-    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+    testCompile 'org.embulk:embulk-standards:0.8.22'
 }

data/src/main/java/org/embulk/output/MySQLOutputPlugin.java

@@ -3,14 +3,21 @@ package org.embulk.output;
 import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
+
 import com.google.common.base.Optional;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.Ssl;
 import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
+import org.embulk.output.mysql.MySQLOutputConnection;
 import org.embulk.output.mysql.MySQLOutputConnector;
 import org.embulk.output.mysql.MySQLBatchInsert;
+import org.embulk.spi.Schema;
 
 public class MySQLOutputPlugin
         extends AbstractJdbcOutputPlugin
@@ -18,6 +25,10 @@ public class MySQLOutputPlugin
     public interface MySQLPluginTask
             extends PluginTask
     {
+        @Config("driver_path")
+        @ConfigDefault("null")
+        public Optional<String> getDriverPath();
+
         @Config("host")
         public String getHost();
 
@@ -34,6 +45,15 @@ public class MySQLOutputPlugin
 
         @Config("database")
         public String getDatabase();
+
+        @Config("temp_database")
+        @ConfigDefault("null")
+        public Optional<String> getTempDatabase();
+
+        @Config("ssl")
+        @ConfigDefault("\"disable\"") // backward compatibility
+        public Ssl getSsl();
+
     }
 
     @Override
@@ -55,6 +75,8 @@ public class MySQLOutputPlugin
     {
         MySQLPluginTask t = (MySQLPluginTask) task;
 
+        loadDriver("com.mysql.jdbc.Driver", t.getDriverPath());
+
         String url = String.format("jdbc:mysql://%s:%d/%s",
                 t.getHost(), t.getPort(), t.getDatabase());
 
@@ -70,21 +92,21 @@ public class MySQLOutputPlugin
         // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
         props.setProperty("tcpKeepAlive", "true");
 
-        // TODO
-        //switch t.getSssl() {
-        //when "disable":
-        //    break;
-        //when "enable":
-        //    props.setProperty("useSSL", "true");
-        //    props.setProperty("requireSSL", "false");
-        //    props.setProperty("verifyServerCertificate", "false");
-        //    break;
-        //when "verify":
-        //    props.setProperty("useSSL", "true");
-        //    props.setProperty("requireSSL", "true");
-        //    props.setProperty("verifyServerCertificate", "true");
-        //    break;
-        //}
+        switch (t.getSsl()) {
+            case DISABLE:
+                props.setProperty("useSSL", "false");
+                break;
+            case ENABLE:
+                props.setProperty("useSSL", "true");
+                props.setProperty("requireSSL", "true");
+                props.setProperty("verifyServerCertificate", "false");
+                break;
+            case VERIFY:
+                props.setProperty("useSSL", "true");
+                props.setProperty("requireSSL", "true");
+                props.setProperty("verifyServerCertificate", "true");
+                break;
+        }
 
         if (!retryableMetadataOperation) {
             // non-retryable batch operation uses longer timeout
@@ -97,12 +119,21 @@ public class MySQLOutputPlugin
         // TODO validate task.getMergeKeys is null
 
         props.setProperty("user", t.getUser());
-        logger.info("Connecting to {} options {}", url, props);
         props.setProperty("password", t.getPassword());
+        logConnectionProperties(url, props);
 
         return new MySQLOutputConnector(url, props);
     }
 
+    @Override
+    protected TableIdentifier buildIntermediateTableId(JdbcOutputConnection con, PluginTask task, String tableName) {
+        MySQLPluginTask t = (MySQLPluginTask) task;
+        if (t.getTempDatabase().isPresent()) {
+            return new TableIdentifier(t.getTempDatabase().get(), null, tableName);
+        }
+        return super.buildIntermediateTableId(con, task, tableName);
+    }
+
     @Override
     protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
@@ -122,4 +153,13 @@ public class MySQLOutputPlugin
                 return false;
         }
     }
+
+    @Override
+    protected void doBegin(JdbcOutputConnection con,
+                           PluginTask task, final Schema schema, int taskCount) throws SQLException
+    {
+        MySQLOutputConnection mySQLCon = (MySQLOutputConnection)con;
+        mySQLCon.compareTimeZone();
+        super.doBegin(con,task,schema,taskCount);
+    }
 }

data/src/main/java/org/embulk/output/MySQLTimeZoneComparison.java

@@ -0,0 +1,107 @@
+package org.embulk.output;
+
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+
+public class MySQLTimeZoneComparison {
+
+    private static final int ONE_HOUR_SEC = 3600;
+    private static final int ONE_MIN_SEC = 60;
+
+    private Connection connection;
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public MySQLTimeZoneComparison(Connection connection)
+    {
+        this.connection = connection;
+    }
+
+    public void compareTimeZone()
+            throws SQLException
+    {
+        TimeZone serverTimeZone = null;
+        try {
+            serverTimeZone = getServerTimeZone();
+        }
+        catch (SQLException ex) {
+            logger.warn("Can't get server TimeZone.");
+            logger.warn(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString()));
+        }
+
+        TimeZone clientTimeZone = TimeZone.getDefault();
+        Date today = new Date();
+        int clientOffset = clientTimeZone.getRawOffset();
+
+        if (clientTimeZone.inDaylightTime(today)) {
+            clientOffset += clientTimeZone.getDSTSavings();
+        }
+
+        //
+        // Compare offset only. Although I expect to return true, the following code return false,
+        //
+        // TimeZone tz_jst   = TimeZone.getTimeZone("JST");
+        // TimeZone tz_gmt9  = TimeZone.getTimeZone("GMT+9");
+        // tz_jst.hasSameRules(tz_gmt9) // return false.
+        //
+        if (clientOffset != serverTimeZone.getRawOffset()) {
+            logger.warn(String.format(Locale.ENGLISH,
+                    "The client timezone(%s) is different from the server timezone(%s). The plugin will store wrong datetime values.",
+                    clientTimeZone.getID(), serverTimeZone.getID()));
+            logger.warn(String.format(Locale.ENGLISH,
+                    "You may need to set options `useLegacyDatetimeCode` and `serverTimezone`"));
+            logger.warn(String.format(Locale.ENGLISH,
+                    "Example. `options: { useLegacyDatetimeCode: false, serverTimezone: UTC }`"));
+        }
+        logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future."));
+    }
+
+    private TimeZone getServerTimeZone()
+            throws SQLException
+    {
+        //
+        // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform.
+        // So, This method calculate GMT offset with query.
+        //
+        String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));";
+        Statement stmt = connection.createStatement();
+
+        try {
+            ResultSet rs = stmt.executeQuery(query);
+            if (rs.next()) {
+                int offsetSeconds = rs.getInt(1);
+                return fromGMTOffsetSeconds(offsetSeconds);
+            }
+            throw new SQLException(String.format(Locale.ENGLISH,
+                    "The timezone comparison query(%s) doesn't return the result.",query));
+        }
+        finally {
+            stmt.close();
+        }
+    }
+
+    private TimeZone fromGMTOffsetSeconds(int offsetSeconds)
+    {
+        if (offsetSeconds == 0) {
+            return TimeZone.getTimeZone("UTC");
+        }
+
+        String sign = offsetSeconds > 0 ? "+" : "-";
+        int absOffsetSec = Math.abs(offsetSeconds);
+        int tzHour = absOffsetSec / ONE_HOUR_SEC;
+        int tzMin = absOffsetSec % ONE_HOUR_SEC / ONE_MIN_SEC;
+        String tzName = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tzHour, tzMin);
+        return TimeZone.getTimeZone(tzName);
+    }
+
+
+}

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java

@@ -4,10 +4,12 @@ import java.util.List;
 import java.sql.Connection;
 import java.sql.SQLException;
 
+import org.embulk.output.MySQLTimeZoneComparison;
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
 
 public class MySQLOutputConnection
         extends JdbcOutputConnection
@@ -20,12 +22,12 @@ public class MySQLOutputConnection
     }
 
     @Override
-    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
+    protected String buildPreparedMergeSql(TableIdentifier toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" (");
         for (int i = 0; i < toTableSchema.getCount(); i++) {
             if(i != 0) { sb.append(", "); }
@@ -58,12 +60,12 @@ public class MySQLOutputConnection
     }
 
     @Override
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
+    protected String buildCollectMergeSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" (");
         for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
@@ -78,7 +80,7 @@ public class MySQLOutputConnection
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
             sb.append(" FROM ");
-            quoteIdentifierString(sb, fromTables.get(i));
+            quoteTableIdentifier(sb, fromTables.get(i));
         }
         sb.append(" ON DUPLICATE KEY UPDATE ");
         if (mergeConfig.getMergeRule().isPresent()) {
@@ -110,4 +112,11 @@ public class MySQLOutputConnection
             return super.buildColumnTypeName(c);
         }
     }
+
+    public void compareTimeZone() throws SQLException
+    {
+        MySQLTimeZoneComparison timeZoneComparison = new MySQLTimeZoneComparison(connection);
+        timeZoneComparison.compareTimeZone();
+    }
+
 }

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java

@@ -1,25 +1,20 @@
 package org.embulk.output.mysql;
 
 import java.util.Properties;
-import java.sql.Driver;
 import java.sql.Connection;
+import java.sql.DriverManager;
 import java.sql.SQLException;
+
 import org.embulk.output.jdbc.JdbcOutputConnector;
 
 public class MySQLOutputConnector
         implements JdbcOutputConnector
 {
-    private final Driver driver;
     private final String url;
     private final Properties properties;
 
     public MySQLOutputConnector(String url, Properties properties)
     {
-        try {
-            this.driver = new com.mysql.jdbc.Driver();  // new com.mysql.jdbc.Driver throws SQLException
-        } catch (SQLException ex) {
-            throw new RuntimeException(ex);
-        }
         this.url = url;
         this.properties = properties;
     }
@@ -27,7 +22,7 @@ public class MySQLOutputConnector
     @Override
     public MySQLOutputConnection connect(boolean autoCommit) throws SQLException
     {
-        Connection c = driver.connect(url, properties);
+        Connection c = DriverManager.getConnection(url, properties);
         try {
             MySQLOutputConnection con = new MySQLOutputConnection(c, autoCommit);
             c = null;

data/src/test/java/org/embulk/output/mysql/AfterLoadTest.java

@@ -0,0 +1,136 @@
+package org.embulk.output.mysql;
+
+import static org.embulk.output.mysql.MySQLTests.execute;
+import static org.embulk.output.mysql.MySQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class AfterLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/after_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_replace_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplaceAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/java/org/embulk/output/mysql/BeforeLoadTest.java

@@ -0,0 +1,124 @@
+package org.embulk.output.mysql;
+
+import static org.embulk.output.mysql.MySQLTests.execute;
+import static org.embulk.output.mysql.MySQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class BeforeLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/before_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}