embulk-output-mysql 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c687ef3905d12e4eb083d8c8e9a94f792dce1682
+  data.tar.gz: c7dbe040a2b18064d4bf271f38a99489f557e32b
+SHA512:
+  metadata.gz: 40b00128822e262fe26428f6810f3a4eb4aaad591216511204b66894237a949a12df411fc37826762e67db7f700dc06d017962915c74434a1ec0a5eb8f9e7474
+  data.tar.gz: cb7f383e3c5c15f9350bab9c7b0410880aace46a6719b6f0e7f86ba1bc54cfbc53994e4c2217232bed02404c4b014c2ee804e46b731f712ddfb02a7b023ac683

data/README.md

@@ -0,0 +1,42 @@
+# MySQL output plugins for Embulk
+
+MySQL output plugins for Embulk loads records to MySQL.
+
+## Overview
+
+* **Plugin type**: output
+* **Load all or nothing**: depnds on the mode:
+  * **insert**: no
+  * **replace**: yes
+* **Resume supported**: no
+
+## Configuration
+
+- **host**: database host name (string, required)
+- **port**: database port number (integer, default: 3306)
+- **user**: database login user name (string, required)
+- **password**: database login password (string, default: "")
+- **database**: destination database name (string, required)
+- **table**: destination name (string, required)
+- **mode**: "replace" or "insert" (string, required)
+- **batch_size**: size of a single batch insert (integer, default: 16777216)
+- **options**: extra connection properties (hash, default: {})
+
+### Example
+
+```yaml
+out:
+  type: mysql
+  host: localhost
+  user: root
+  password: ""
+  database: my_database
+  table: my_table
+  mode: insert
+```
+
+### Build
+
+```
+$ ./gradlew gem
+```

data/build.gradle

@@ -0,0 +1,7 @@
+dependencies {
+    compile project(':embulk-output-jdbc')
+
+    compile 'mysql:mysql-connector-java:5.1.34'
+
+    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+}

data/lib/embulk/output/mysql.rb

@@ -0,0 +1,3 @@
+Embulk::JavaPlugin.register_output(
+  :mysql, "org.embulk.output.MySQLOutputPlugin",
+  File.expand_path('../../../../classpath', __FILE__))

data/src/main/java/org/embulk/output/MySQLOutputPlugin.java

@@ -0,0 +1,72 @@
+package org.embulk.output;
+
+import java.util.Properties;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.sql.Connection;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.mysql.MySQLOutputConnector;
+import org.embulk.output.mysql.MySQLBatchInsert;
+
+public class MySQLOutputPlugin
+        extends AbstractJdbcOutputPlugin
+{
+    private static final int DEFAULT_PORT = 3306;
+
+    @Override
+    protected MySQLOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    {
+        String url = String.format("jdbc:mysql://%s:%d/%s",
+                task.getHost(), task.getPort().or(DEFAULT_PORT), task.getDatabase());
+
+        Properties props = new Properties();
+        props.setProperty("user", task.getUser());
+        props.setProperty("password", task.getPassword());
+
+        props.setProperty("rewriteBatchedStatements", "true");
+        props.setProperty("useCompression", "true");
+
+        props.setProperty("connectTimeout", "300000"); // milliseconds
+        props.setProperty("socketTimeout", "1800000"); // smillieconds
+
+        // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
+        // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.
+        props.setProperty("tcpKeepAlive", "true");
+
+        // TODO
+        //switch task.getSssl() {
+        //when "disable":
+        //    break;
+        //when "enable":
+        //    props.setProperty("useSSL", "true");
+        //    props.setProperty("requireSSL", "false");
+        //    props.setProperty("verifyServerCertificate", "false");
+        //    break;
+        //when "verify":
+        //    props.setProperty("useSSL", "true");
+        //    props.setProperty("requireSSL", "true");
+        //    props.setProperty("verifyServerCertificate", "true");
+        //    break;
+        //}
+
+        if (!retryableMetadataOperation) {
+            // non-retryable batch operation uses longer timeout
+            props.setProperty("connectTimeout",  "300000");  // milliseconds
+            props.setProperty("socketTimeout", "2700000");   // milliseconds
+        }
+
+        props.putAll(task.getOptions());
+
+        return new MySQLOutputConnector(url, props);
+    }
+
+    @Override
+    protected BatchInsert newBatchInsert(PluginTask task) throws IOException, SQLException
+    {
+        return new MySQLBatchInsert(getConnector(task, true));
+    }
+}

data/src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java

@@ -0,0 +1,36 @@
+package org.embulk.output.mysql;
+
+import java.io.IOException;
+import java.sql.Types;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import org.embulk.output.jdbc.StandardBatchInsert;
+
+public class MySQLBatchInsert
+        extends StandardBatchInsert
+{
+    public MySQLBatchInsert(MySQLOutputConnector connector) throws IOException, SQLException
+    {
+        super(connector);
+    }
+
+    @Override
+    public void setFloat(float v) throws IOException, SQLException
+    {
+        if (Float.isNaN(v) || Float.isInfinite(v)) {
+            setNull(Types.REAL);  // TODO get through argument
+        } else {
+            super.setFloat(v);
+        }
+    }
+
+    @Override
+    public void setDouble(double v) throws IOException, SQLException
+    {
+        if (Double.isNaN(v) || Double.isInfinite(v)) {
+            setNull(Types.DOUBLE);  // TODO get through argument
+        } else {
+            super.setDouble(v);
+        }
+    }
+}

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java

@@ -0,0 +1,30 @@
+package org.embulk.output.mysql;
+
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.embulk.spi.Exec;
+import org.embulk.output.jdbc.BatchInsert;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcColumn;
+
+public class MySQLOutputConnection
+        extends JdbcOutputConnection
+{
+    public MySQLOutputConnection(Connection connection, boolean autoCommit)
+            throws SQLException
+    {
+        super(connection, null);
+        connection.setAutoCommit(autoCommit);
+    }
+
+    @Override
+    protected String convertTypeName(String typeName)
+    {
+        switch(typeName) {
+        case "CLOB":
+            return "TEXT";
+        default:
+            return typeName;
+        }
+    }
+}

data/src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java

@@ -0,0 +1,42 @@
+package org.embulk.output.mysql;
+
+import java.util.Properties;
+import java.sql.Driver;
+import java.sql.Connection;
+import java.sql.SQLException;
+import org.embulk.output.jdbc.JdbcOutputConnector;
+import org.embulk.output.jdbc.JdbcOutputConnection;
+
+public class MySQLOutputConnector
+        implements JdbcOutputConnector
+{
+    private final Driver driver;
+    private final String url;
+    private final Properties properties;
+
+    public MySQLOutputConnector(String url, Properties properties)
+    {
+        try {
+            this.driver = new com.mysql.jdbc.Driver();  // new com.mysql.jdbc.Driver throws SQLException
+        } catch (SQLException ex) {
+            throw new RuntimeException(ex);
+        }
+        this.url = url;
+        this.properties = properties;
+    }
+
+    @Override
+    public MySQLOutputConnection connect(boolean autoCommit) throws SQLException
+    {
+        Connection c = driver.connect(url, properties);
+        try {
+            MySQLOutputConnection con = new MySQLOutputConnection(c, autoCommit);
+            c = null;
+            return con;
+        } finally {
+            if (c != null) {
+                c.close();
+            }
+        }
+    }
+}

metadata

@@ -0,0 +1,54 @@
+--- !ruby/object:Gem::Specification
+name: embulk-output-mysql
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- FURUHASHI Sadayuki
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-02-16 00:00:00.000000000 Z
+dependencies: []
+description: JDBC output plugin is an Embulk plugin that loads records to JDBC read by any input plugins. Search the input plugins by "embulk-input" keyword.
+email:
+- frsyuki@users.sourceforge.jp
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- build.gradle
+- lib/embulk/output/mysql.rb
+- src/main/java/org/embulk/output/MySQLOutputPlugin.java
+- src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java
+- src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java
+- src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java
+- classpath/embulk-output-jdbc-0.1.0.jar
+- classpath/embulk-output-mysql-0.1.0.jar
+- classpath/mysql-connector-java-5.1.34.jar
+homepage: https://github.com/embulk/embulk-output-jdbc
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: JDBC output plugin for Embulk
+test_files: []