embulk-output-mysql 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec057792c6f84dbf9922a6d449e42224b83821c8
-  data.tar.gz: 511c6a404700f429ad9601aad2acb877b275843e
+  metadata.gz: 753cd35ceccb305741025c9f2956e2a8bcd43e0f
+  data.tar.gz: a264b7fe51085f24b4a96bfe3f98defc45f3ba4f
 SHA512:
-  metadata.gz: a71c7ef498318bc1857c4845e4826d5d33208cee89a7cfb16f991f5a3748351c3bf84bee0d59fc8b791e93031ba3b4a38727f7d8aad7be5098cc43acdc5c7747
-  data.tar.gz: 58e96c01b69b017fe1f5ab1765e75d799ba90af2a6ec0703e775e52f038760dd793ea14950533ce36b0b269ea86ee1e8b5546550b91a27411cd0daa08c21a3b4
+  metadata.gz: e3af007fe65e593ab4ec3740a3a399d9c622e6867af811c2774445744ebe0a972eddc7610c692793a81a8b9eab7e086bc444ef796fb1022c1e686a3ff1ad5343
+  data.tar.gz: 9e59a1bf4698cca86fc4389c66e84b6a4ad17cc778fc4c72fd55f43377120479fc369f2d91e3247867d00b2f62578624abd368dafa34495c432c01a94e1250b6

data/README.md

@@ -21,8 +21,9 @@ MySQL output plugin for Embulk loads records to MySQL.
 - **table**: destination table name (string, required)
 - **create_table_constraint**: table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **create_table_option**: table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **transaction_isolation**: transaction isolation level for each connection ("read_uncommitted", "read_committed", "repeatable_read" or "serializable"). if not specified, database default value will be used.
 - **options**: extra connection properties (hash, default: {})
-- **retry_limit**: max retry count for database operations (integer, default: 12)
+- **retry_limit**: max retry count for database operations (integer, default: 12). When intermediate table to create already created by another process, this plugin will retry with another table name to avoid collision. And, when a deadlock occurs in loading records, this plugin will retry loading after the transaction rolled back.
 - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", "merge", "merge_direct", or "replace". See below. (string, required)

data/src/main/java/org/embulk/output/MySQLOutputPlugin.java

@@ -12,6 +12,7 @@ import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.Ssl;
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcOutputConnector;
 import org.embulk.output.jdbc.MergeConfig;
 import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.output.mysql.MySQLOutputConnection;
@@ -71,7 +72,7 @@ public class MySQLOutputPlugin
     }
 
     @Override
-    protected MySQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
     {
         MySQLPluginTask t = (MySQLPluginTask) task;
 
@@ -122,7 +123,7 @@ public class MySQLOutputPlugin
         props.setProperty("password", t.getPassword());
         logConnectionProperties(url, props);
 
-        return new MySQLOutputConnector(url, props);
+        return new MySQLOutputConnector(url, props, task.getTransactionIsolation());
     }
 
     @Override

data/src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java

@@ -1,17 +1,17 @@
 package org.embulk.output.mysql;
 
-import java.util.List;
 import java.io.IOException;
 import java.sql.Types;
 import java.sql.SQLException;
 import com.google.common.base.Optional;
+import org.embulk.output.jdbc.JdbcOutputConnector;
 import org.embulk.output.jdbc.MergeConfig;
 import org.embulk.output.jdbc.StandardBatchInsert;
 
 public class MySQLBatchInsert
         extends StandardBatchInsert
 {
-    public MySQLBatchInsert(MySQLOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
+    public MySQLBatchInsert(JdbcOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
         super(connector, mergeConfig);
     }

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java

@@ -14,11 +14,10 @@ import org.embulk.output.jdbc.TableIdentifier;
 public class MySQLOutputConnection
         extends JdbcOutputConnection
 {
-    public MySQLOutputConnection(Connection connection, boolean autoCommit)
+    public MySQLOutputConnection(Connection connection)
             throws SQLException
     {
         super(connection, null);
-        connection.setAutoCommit(autoCommit);
     }
 
     @Override

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java

@@ -5,26 +5,32 @@ import java.sql.Connection;
 import java.sql.DriverManager;
 import java.sql.SQLException;
 
-import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.AbstractJdbcOutputConnector;
+import org.embulk.output.jdbc.TransactionIsolation;
+
+import com.google.common.base.Optional;
 
 public class MySQLOutputConnector
-        implements JdbcOutputConnector
+        extends AbstractJdbcOutputConnector
 {
     private final String url;
     private final Properties properties;
 
-    public MySQLOutputConnector(String url, Properties properties)
+    public MySQLOutputConnector(String url, Properties properties,
+            Optional<TransactionIsolation> transactionIsolation)
     {
+        super(transactionIsolation);
         this.url = url;
         this.properties = properties;
     }
 
     @Override
-    public MySQLOutputConnection connect(boolean autoCommit) throws SQLException
+    protected JdbcOutputConnection connect() throws SQLException
     {
         Connection c = DriverManager.getConnection(url, properties);
         try {
-            MySQLOutputConnection con = new MySQLOutputConnection(c, autoCommit);
+            MySQLOutputConnection con = new MySQLOutputConnection(c);
             c = null;
             return con;
         } finally {

data/src/test/java/org/embulk/output/mysql/MySQLTests.java

@@ -6,6 +6,9 @@ import static org.embulk.test.EmbulkTests.readSortedFile;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
 
 import org.embulk.config.ConfigSource;
 import org.embulk.test.EmbulkTests;
@@ -22,6 +25,17 @@ public class MySQLTests
         return EmbulkTests.config("EMBULK_OUTPUT_MYSQL_TEST_CONFIG");
     }
 
+    public static Connection connect() throws SQLException
+    {
+        ConfigSource config = baseConfig();
+
+        String url = String.format("jdbc:mysql://%s:%s/%s",
+                config.get(String.class, "host"),
+                config.get(String.class, "port", "3306"),
+                config.get(String.class, "database"));
+        return DriverManager.getConnection(url, config.get(String.class, "user"), config.get(String.class, "password"));
+    }
+
     public static void execute(String sql)
     {
         ConfigSource config = baseConfig();

data/src/test/java/org/embulk/output/mysql/RetryTest.java

@@ -0,0 +1,139 @@
+package org.embulk.output.mysql;
+
+import static org.embulk.output.mysql.MySQLTests.execute;
+import static org.embulk.output.mysql.MySQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+import java.sql.Connection;
+import java.sql.Statement;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class RetryTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/retry/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testSucceeded() throws Exception
+    {
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test1.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test1_expected.csv")));
+    }
+
+    @Test
+    public void testRetry() throws Exception
+    {
+        Thread thread = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 100; i < 110; i++) {
+                                statement.execute("insert into test1 values('B" + i + "', 0)");
+                            }
+
+                            statement.execute("insert into test1 values('A003', 0)");
+                            Thread.sleep(3000);
+                            // deadlock will occur
+                            statement.execute("insert into test1 values('A002', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread.start();
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test1.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test1_expected.csv")));
+    }
+
+    // will be flushed multiple times
+    @Test
+    public void testRetryLarge() throws Exception
+    {
+        Thread thread = new Thread() {
+            @Override
+            public void run() {
+                try {
+                    try (Connection conn = MySQLTests.connect()) {
+                        conn.setAutoCommit(false);
+                        try (Statement statement = conn.createStatement()) {
+                            // make the transaction larger so that embulk-output-mysql transaction will be rolled back at a deadlock.
+                            for (int i = 100; i < 200; i++) {
+                                statement.execute("insert into test1 values('B" + i + "', 0)");
+                            }
+
+                            statement.execute("insert into test1 values('A170', 0)");
+                            Thread.sleep(3000);
+                            // deadlock will occur
+                            statement.execute("insert into test1 values('A160', 0)");
+                            conn.rollback();
+                        }
+                    }
+                } catch (Exception e) {
+                    e.printStackTrace();
+                }
+            }
+        };
+        thread.start();
+
+        Path in1 = toPath("test1_large.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test1_large.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test1_large_expected.csv")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/setup.sql

@@ -0,0 +1,7 @@
+drop table if exists test1;
+
+create table test1 (
+    id              char(4),
+    num             int,
+    primary key (id)
+);

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.csv

@@ -0,0 +1,5 @@
+id:string,num:long
+A001,11
+A002,22
+A003,33
+A004,44

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.yml

@@ -0,0 +1,2 @@
+table: test1
+mode: insert_direct

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_expected.csv

@@ -0,0 +1,4 @@
+A001,11
+A002,22
+A003,33
+A004,44

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large.csv

@@ -0,0 +1,101 @@
+id:string,num:long
+A100,100
+A101,101
+A102,102
+A103,103
+A104,104
+A105,105
+A106,106
+A107,107
+A108,108
+A109,109
+A110,110
+A111,111
+A112,112
+A113,113
+A114,114
+A115,115
+A116,116
+A117,117
+A118,118
+A119,119
+A120,120
+A121,121
+A122,122
+A123,123
+A124,124
+A125,125
+A126,126
+A127,127
+A128,128
+A129,129
+A130,130
+A131,131
+A132,132
+A133,133
+A134,134
+A135,135
+A136,136
+A137,137
+A138,138
+A139,139
+A140,140
+A141,141
+A142,142
+A143,143
+A144,144
+A145,145
+A146,146
+A147,147
+A148,148
+A149,149
+A150,150
+A151,151
+A152,152
+A153,153
+A154,154
+A155,155
+A156,156
+A157,157
+A158,158
+A159,159
+A160,160
+A161,161
+A162,162
+A163,163
+A164,164
+A165,165
+A166,166
+A167,167
+A168,168
+A169,169
+A170,170
+A171,171
+A172,172
+A173,173
+A174,174
+A175,175
+A176,176
+A177,177
+A178,178
+A179,179
+A180,180
+A181,181
+A182,182
+A183,183
+A184,184
+A185,185
+A186,186
+A187,187
+A188,188
+A189,189
+A190,190
+A191,191
+A192,192
+A193,193
+A194,194
+A195,195
+A196,196
+A197,197
+A198,198
+A199,199

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: insert_direct
+batch_size: 1000

data/src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large_expected.csv

@@ -0,0 +1,100 @@
+A100,100
+A101,101
+A102,102
+A103,103
+A104,104
+A105,105
+A106,106
+A107,107
+A108,108
+A109,109
+A110,110
+A111,111
+A112,112
+A113,113
+A114,114
+A115,115
+A116,116
+A117,117
+A118,118
+A119,119
+A120,120
+A121,121
+A122,122
+A123,123
+A124,124
+A125,125
+A126,126
+A127,127
+A128,128
+A129,129
+A130,130
+A131,131
+A132,132
+A133,133
+A134,134
+A135,135
+A136,136
+A137,137
+A138,138
+A139,139
+A140,140
+A141,141
+A142,142
+A143,143
+A144,144
+A145,145
+A146,146
+A147,147
+A148,148
+A149,149
+A150,150
+A151,151
+A152,152
+A153,153
+A154,154
+A155,155
+A156,156
+A157,157
+A158,158
+A159,159
+A160,160
+A161,161
+A162,162
+A163,163
+A164,164
+A165,165
+A166,166
+A167,167
+A168,168
+A169,169
+A170,170
+A171,171
+A172,172
+A173,173
+A174,174
+A175,175
+A176,176
+A177,177
+A178,178
+A179,179
+A180,180
+A181,181
+A182,182
+A183,183
+A184,184
+A185,185
+A186,186
+A187,187
+A188,188
+A189,189
+A190,190
+A191,191
+A192,192
+A193,193
+A194,194
+A195,195
+A196,196
+A197,197
+A198,198
+A199,199

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mysql
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-30 00:00:00.000000000 Z
+date: 2019-05-07 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.8.2.jar
-- classpath/embulk-output-mysql-0.8.2.jar
+- classpath/embulk-output-jdbc-0.8.3.jar
+- classpath/embulk-output-mysql-0.8.3.jar
 - default_jdbc_driver/mysql-connector-java-5.1.44.jar
 - lib/embulk/output/mysql.rb
 - src/main/java/org/embulk/output/MySQLOutputPlugin.java
@@ -32,6 +32,7 @@ files:
 - src/test/java/org/embulk/output/mysql/BeforeLoadTest.java
 - src/test/java/org/embulk/output/mysql/CreateTableTest.java
 - src/test/java/org/embulk/output/mysql/MySQLTests.java
+- src/test/java/org/embulk/output/mysql/RetryTest.java
 - src/test/resources/org/embulk/output/mysql/test/expect/after_load/setup.sql
 - src/test/resources/org/embulk/output/mysql/test/expect/after_load/test1.csv
 - src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_expected.diff
@@ -62,6 +63,13 @@ files:
 - src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_constraint_expected.csv
 - src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_option.yml
 - src/test/resources/org/embulk/output/mysql/test/expect/create_table/test_table_option_expected.csv
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/setup.sql
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.csv
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1.yml
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_expected.csv
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large.csv
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large.yml
+- src/test/resources/org/embulk/output/mysql/test/expect/retry/test1_large_expected.csv
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0