embulk-output-mysql 0.7.8 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 97ce26c6892e8428a757a29cfcf1635d37d5ed2e
-  data.tar.gz: 6daf806203e0a6204cd3f69f4b34f012422cd3a6
+  metadata.gz: 08449007dfa072591724efe96735f7f36e47890f
+  data.tar.gz: 39acd32a644c1a5dfec133f5d5d5705d6653b6a1
 SHA512:
-  metadata.gz: b737ebe4debbc5657a998fbceb3e3d5355ea9a041a2d8a6c5193a68611f03968a888f79c71f13a5a024385818ead1ec9e180403511608fc735aeaf42a61e36bf
-  data.tar.gz: 40a5b20c6126b885394eaf7e883a7f4be05ef18396e79946bef23a77766bf6c3422571b8b774ee697dc7ac1efdfd133eea189efe8bd7bee405ab5f8ef5534ff4
+  metadata.gz: fdc887a07cda0ab68e9e00013ea3c0be5e480647728476a9f0ae9f847bc197ce12684d120eea627f622af54efdba7d38baf94b9f0202a2c9f4e30fc9bf736201
+  data.tar.gz: b5f443bdc839934d3c2db7d4f8fb699eb50b2b8ef618c8eb2efd2a6d491db4fdaf6fa16dcde7a75d8d31302bd1a8e12e5cd921846bc701ee3de9845ddbd2a322

data/README.md

@@ -15,6 +15,7 @@ MySQL output plugin for Embulk loads records to MySQL.
 - **user**: database login user name (string, required)
 - **password**: database login password (string, default: "")
 - **database**: destination database name (string, required)
+- **temp_database**: database name for intermediate tables. by default, intermediate tables will be created in the database specified by `database`. (string, optional)
 - **table**: destination table name (string, required)
 - **options**: extra connection properties (hash, default: {})
 - **retry_limit** max retry count for database operations (integer, default: 12)

data/src/main/java/org/embulk/output/MySQLOutputPlugin.java

@@ -3,14 +3,20 @@ package org.embulk.output;
 import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
+
 import com.google.common.base.Optional;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
+import org.embulk.output.mysql.MySQLOutputConnection;
 import org.embulk.output.mysql.MySQLOutputConnector;
 import org.embulk.output.mysql.MySQLBatchInsert;
+import org.embulk.spi.Schema;
 
 public class MySQLOutputPlugin
         extends AbstractJdbcOutputPlugin
@@ -34,6 +40,10 @@ public class MySQLOutputPlugin
 
         @Config("database")
         public String getDatabase();
+
+        @Config("temp_database")
+        @ConfigDefault("null")
+        public Optional<String> getTempDatabase();
     }
 
     @Override
@@ -103,6 +113,15 @@ public class MySQLOutputPlugin
         return new MySQLOutputConnector(url, props);
     }
 
+    @Override
+    protected TableIdentifier buildIntermediateTableId(JdbcOutputConnection con, PluginTask task, String tableName) {
+        MySQLPluginTask t = (MySQLPluginTask) task;
+        if (t.getTempDatabase().isPresent()) {
+            return new TableIdentifier(t.getTempDatabase().get(), null, tableName);
+        }
+        return super.buildIntermediateTableId(con, task, tableName);
+    }
+
     @Override
     protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
@@ -122,4 +141,13 @@ public class MySQLOutputPlugin
                 return false;
         }
     }
+
+    @Override
+    protected void doBegin(JdbcOutputConnection con,
+                           PluginTask task, final Schema schema, int taskCount) throws SQLException
+    {
+        MySQLOutputConnection mySQLCon = (MySQLOutputConnection)con;
+        mySQLCon.compareTimeZone();
+        super.doBegin(con,task,schema,taskCount);
+    }
 }

data/src/main/java/org/embulk/output/MySQLTimeZoneComparison.java

@@ -0,0 +1,107 @@
+package org.embulk.output;
+
+import org.embulk.spi.Exec;
+import org.slf4j.Logger;
+
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Date;
+import java.util.Locale;
+import java.util.TimeZone;
+
+public class MySQLTimeZoneComparison {
+
+    private static final int ONE_HOUR_SEC = 3600;
+    private static final int ONE_MIN_SEC = 60;
+
+    private Connection connection;
+
+    private final Logger logger = Exec.getLogger(getClass());
+
+    public MySQLTimeZoneComparison(Connection connection)
+    {
+        this.connection = connection;
+    }
+
+    public void compareTimeZone()
+            throws SQLException
+    {
+        TimeZone serverTimeZone = null;
+        try {
+            serverTimeZone = getServerTimeZone();
+        }
+        catch (SQLException ex) {
+            logger.warn("Can't get server TimeZone.");
+            logger.warn(String.format(Locale.ENGLISH, "SQLException raised %s", ex.toString()));
+        }
+
+        TimeZone clientTimeZone = TimeZone.getDefault();
+        Date today = new Date();
+        int clientOffset = clientTimeZone.getRawOffset();
+
+        if (clientTimeZone.inDaylightTime(today)) {
+            clientOffset += clientTimeZone.getDSTSavings();
+        }
+
+        //
+        // Compare offset only. Although I expect to return true, the following code return false,
+        //
+        // TimeZone tz_jst   = TimeZone.getTimeZone("JST");
+        // TimeZone tz_gmt9  = TimeZone.getTimeZone("GMT+9");
+        // tz_jst.hasSameRules(tz_gmt9) // return false.
+        //
+        if (clientOffset != serverTimeZone.getRawOffset()) {
+            logger.warn(String.format(Locale.ENGLISH,
+                    "The client timezone(%s) is different from the server timezone(%s). The plugin will store wrong datetime values.",
+                    clientTimeZone.getID(), serverTimeZone.getID()));
+            logger.warn(String.format(Locale.ENGLISH,
+                    "You may need to set options `useLegacyDatetimeCode` and `serverTimezone`"));
+            logger.warn(String.format(Locale.ENGLISH,
+                    "Example. `options: { useLegacyDatetimeCode: false, serverTimezone: UTC }`"));
+        }
+        logger.warn(String.format(Locale.ENGLISH, "The plugin will set `useLegacyDatetimeCode=false` by default in future."));
+    }
+
+    private TimeZone getServerTimeZone()
+            throws SQLException
+    {
+        //
+        // First, I used `@@system_time_zone`. but It return non Time Zone Abbreviations name on a specific platform.
+        // So, This method calculate GMT offset with query.
+        //
+        String query = "select TIME_TO_SEC(timediff(now(),utc_timestamp()));";
+        Statement stmt = connection.createStatement();
+
+        try {
+            ResultSet rs = stmt.executeQuery(query);
+            if (rs.next()) {
+                int offsetSeconds = rs.getInt(1);
+                return fromGMTOffsetSeconds(offsetSeconds);
+            }
+            throw new SQLException(String.format(Locale.ENGLISH,
+                    "The timezone comparison query(%s) doesn't return the result.",query));
+        }
+        finally {
+            stmt.close();
+        }
+    }
+
+    private TimeZone fromGMTOffsetSeconds(int offsetSeconds)
+    {
+        if (offsetSeconds == 0) {
+            return TimeZone.getTimeZone("UTC");
+        }
+
+        String sign = offsetSeconds > 0 ? "+" : "-";
+        int absOffsetSec = Math.abs(offsetSeconds);
+        int tzHour = absOffsetSec / ONE_HOUR_SEC;
+        int tzMin = absOffsetSec % ONE_HOUR_SEC / ONE_MIN_SEC;
+        String tzName = String.format(Locale.ENGLISH, "GMT%s%02d:%02d", sign, tzHour, tzMin);
+        return TimeZone.getTimeZone(tzName);
+    }
+
+
+}

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java

@@ -4,10 +4,12 @@ import java.util.List;
 import java.sql.Connection;
 import java.sql.SQLException;
 
+import org.embulk.output.MySQLTimeZoneComparison;
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
 
 public class MySQLOutputConnection
         extends JdbcOutputConnection
@@ -20,12 +22,12 @@ public class MySQLOutputConnection
     }
 
     @Override
-    protected String buildPreparedMergeSql(String toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
+    protected String buildPreparedMergeSql(TableIdentifier toTable, JdbcSchema toTableSchema, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" (");
         for (int i = 0; i < toTableSchema.getCount(); i++) {
             if(i != 0) { sb.append(", "); }
@@ -58,12 +60,12 @@ public class MySQLOutputConnection
     }
 
     @Override
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
+    protected String buildCollectMergeSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("INSERT INTO ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" (");
         for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
@@ -78,7 +80,7 @@ public class MySQLOutputConnection
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
             sb.append(" FROM ");
-            quoteIdentifierString(sb, fromTables.get(i));
+            quoteTableIdentifier(sb, fromTables.get(i));
         }
         sb.append(" ON DUPLICATE KEY UPDATE ");
         if (mergeConfig.getMergeRule().isPresent()) {
@@ -110,4 +112,11 @@ public class MySQLOutputConnection
             return super.buildColumnTypeName(c);
         }
     }
+
+    public void compareTimeZone() throws SQLException
+    {
+        MySQLTimeZoneComparison timeZoneComparison = new MySQLTimeZoneComparison(connection);
+        timeZoneComparison.compareTimeZone();
+    }
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mysql
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-10 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,11 +19,12 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.8.jar
-- classpath/embulk-output-mysql-0.7.8.jar
+- classpath/embulk-output-jdbc-0.7.9.jar
+- classpath/embulk-output-mysql-0.7.9.jar
 - classpath/mysql-connector-java-5.1.34.jar
 - lib/embulk/output/mysql.rb
 - src/main/java/org/embulk/output/MySQLOutputPlugin.java
+- src/main/java/org/embulk/output/MySQLTimeZoneComparison.java
 - src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java