embulk-output-mysql 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 03bd38c5d7d0940e028927fc85e85bec5b73bb24
-  data.tar.gz: 08e939d20b51d4e3299740910846a6e89cd621d9
+  metadata.gz: 069acac0c6d408b6e2c2b4c4e95a1fec7a38be4c
+  data.tar.gz: 4235adc1b6848f48e3f22e0d97098508c5ffa929
 SHA512:
-  metadata.gz: eb728ffff43b05e68c10208aa71df6a48a219945f88f9babebd22d17c6302826d962fcdf98b2fd5ef6f67e4418c7d0a0650110eabfd8ab21276cfdf9ff170aff
-  data.tar.gz: 62b977143db7a65a527f7e316fa8a1f0fe5354eeb1cb3d3afb704d7e08f70f67e6f1c02137d1058fb6c017146173558d3596a12db2250e8c7ddc9283dbd6e5e8
+  metadata.gz: 54f83241c5c37fa5998d91e60b7c1cd2f1534a86a0ecddc58889093656477989bf28ca51124e744953687de867716903df259ca5dbd80e3146c869ef2ad728de
+  data.tar.gz: 3ee5327bd2930e5e17a8ee5ea3df9234afe58166be2e59c997943eeae11aed07be136d936b19215404dcb04dfefcf8babf81656731c773e214eab9cf59e56c37

data/README.md

@@ -5,8 +5,8 @@ MySQL output plugins for Embulk loads records to MySQL.
 ## Overview
 
 * **Plugin type**: output
-* **Load all or nothing**: depnds on the mode. see below.
-* **Resume supported**: depnds on the mode. see below.
+* **Load all or nothing**: depends on the mode. see below.
+* **Resume supported**: depends on the mode. see below.
 
 ## Configuration
 
@@ -17,7 +17,11 @@ MySQL output plugins for Embulk loads records to MySQL.
 - **database**: destination database name (string, required)
 - **table**: destination table name (string, required)
 - **options**: extra connection properties (hash, default: {})
+- **retry_limit** max retry count for database operations (integer, default: 12)
+- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
+- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", "merge", "merge_direct", or "replace". See below. (string, required)
+- **merge_rule**: list of column assignments for updating existing records used in merge and merge_direct modes, for example `foo = target_table.foo + VALUES(foo)` in case of merge mode, or `foo = foo + VALUES(foo)` in case of merge_direct mode. (string array, default: always overwrites with new values)
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.

data/src/main/java/org/embulk/output/MySQLOutputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.output;
 
-import java.util.List;
 import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
@@ -9,6 +8,7 @@ import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.MergeConfig;
 import org.embulk.output.mysql.MySQLOutputConnector;
 import org.embulk.output.mysql.MySQLBatchInsert;
 
@@ -104,8 +104,22 @@ public class MySQLOutputPlugin
     }
 
     @Override
-    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
-        return new MySQLBatchInsert(getConnector(task, true), mergeKeys);
+        return new MySQLBatchInsert(getConnector(task, true), mergeConfig);
+    }
+
+
+    @Override
+    protected boolean isRetryableException(String sqlState, int errorCode)
+    {
+        switch (errorCode) {
+            case 1213: // ER_LOCK_DEADLOCK (Message: Deadlock found when trying to get lock; try restarting transaction)
+                return true;
+            case 1205: // ER_LOCK_WAIT_TIMEOUT (Message: Lock wait timeout exceeded; try restarting transaction)
+                return true;
+            default:
+                return false;
+        }
     }
 }

data/src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java

@@ -5,14 +5,15 @@ import java.io.IOException;
 import java.sql.Types;
 import java.sql.SQLException;
 import com.google.common.base.Optional;
+import org.embulk.output.jdbc.MergeConfig;
 import org.embulk.output.jdbc.StandardBatchInsert;
 
 public class MySQLBatchInsert
         extends StandardBatchInsert
 {
-    public MySQLBatchInsert(MySQLOutputConnector connector, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    public MySQLBatchInsert(MySQLOutputConnector connector, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
-        super(connector, mergeKeys);
+        super(connector, mergeConfig);
     }
 
     @Override

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java

@@ -3,9 +3,11 @@ package org.embulk.output.mysql;
 import java.util.List;
 import java.sql.Connection;
 import java.sql.SQLException;
+
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.MergeConfig;
 
 public class MySQLOutputConnection
         extends JdbcOutputConnection
@@ -18,50 +20,60 @@ public class MySQLOutputConnection
     }
 
     @Override
-    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, List<String> mergeKeys) throws SQLException
+    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
         quoteIdentifierString(sb, toTable);
         sb.append(" (");
-        for (int i=0; i < toTableSchema.getCount(); i++) {
+        for (int i = 0; i < toTableSchema.getCount(); i++) {
             if(i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, toTableSchema.getColumnName(i));
         }
         sb.append(") VALUES (");
-        for(int i=0; i < toTableSchema.getCount(); i++) {
+        for(int i = 0; i < toTableSchema.getCount(); i++) {
             if(i != 0) { sb.append(", "); }
             sb.append("?");
         }
         sb.append(")");
         sb.append(" ON DUPLICATE KEY UPDATE ");
-        for (int i=0; i < toTableSchema.getCount(); i++) {
-            if(i != 0) { sb.append(", "); }
-            String columnName = quoteIdentifierString(toTableSchema.getColumnName(i));
-            sb.append(columnName).append(" = VALUES(").append(columnName).append(")");
+        if (mergeConfig.getMergeRule().isPresent()) {
+            List<String> rule = mergeConfig.getMergeRule().get();
+            for (int i = 0; i < rule.size(); i++) {
+                if (i != 0) {
+                    sb.append(", ");
+                }
+                sb.append(rule.get(i));
+            }
+        } else {
+            for (int i = 0; i < toTableSchema.getCount(); i++) {
+                if(i != 0) { sb.append(", "); }
+                String columnName = quoteIdentifierString(toTableSchema.getColumnName(i));
+                sb.append(columnName).append(" = VALUES(").append(columnName).append(")");
+            }
         }
 
         return sb.toString();
     }
 
     @Override
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
         quoteIdentifierString(sb, toTable);
         sb.append(" (");
-        for (int i=0; i < schema.getCount(); i++) {
+        for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, schema.getColumnName(i));
         }
         sb.append(") ");
-        for (int i=0; i < fromTables.size(); i++) {
+        for (int i = 0; i < fromTables.size(); i++) {
             if (i != 0) { sb.append(" UNION ALL "); }
             sb.append("SELECT ");
-            for (int j=0; j < schema.getCount(); j++) {
+            for (int j = 0; j < schema.getCount(); j++) {
                 if (j != 0) { sb.append(", "); }
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
@@ -69,10 +81,20 @@ public class MySQLOutputConnection
             quoteIdentifierString(sb, fromTables.get(i));
         }
         sb.append(" ON DUPLICATE KEY UPDATE ");
-        for (int i=0; i < schema.getCount(); i++) {
-            if(i != 0) { sb.append(", "); }
-            String columnName = quoteIdentifierString(schema.getColumnName(i));
-            sb.append(columnName).append(" = VALUES(").append(columnName).append(")");
+        if (mergeConfig.getMergeRule().isPresent()) {
+            List<String> rule = mergeConfig.getMergeRule().get();
+            for (int i = 0; i < rule.size(); i++) {
+                if (i != 0) {
+                    sb.append(", ");
+                }
+                sb.append(rule.get(i));
+            }
+        } else {
+            for (int i = 0; i < schema.getCount(); i++) {
+                if(i != 0) { sb.append(", "); }
+                String columnName = quoteIdentifierString(schema.getColumnName(i));
+                sb.append(columnName).append(" = VALUES(").append(columnName).append(")");
+            }
         }
 
         return sb.toString();

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mysql
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-23 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -24,8 +24,8 @@ files:
 - src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java
-- classpath/embulk-output-jdbc-0.6.1.jar
-- classpath/embulk-output-mysql-0.6.1.jar
+- classpath/embulk-output-jdbc-0.6.2.jar
+- classpath/embulk-output-mysql-0.6.2.jar
 - classpath/mysql-connector-java-5.1.34.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses: