embulk-input-mysql 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4f416fe2b7b01cb603bcdc330f0d956988f2756b
-  data.tar.gz: 73e22f9633062e6f85bd55c1e231295c332acaaa
+  metadata.gz: a09bee0f3126d679f39ab5f84f1eff085565f573
+  data.tar.gz: 216094c890843f89ede7e06717029f729e2d3f74
 SHA512:
-  metadata.gz: 4d4519bc01ab852dc20dd8ed219c460b163a524c097e9362e80575ffbb3e3b18bb989c8b2eeb4908eb3c535b36278d4537f6d8d0135b22eea3335f520e0abe94
-  data.tar.gz: 5c2b2044aa61697b7fac315d7077838267d24f56147ca0f31f9d039ef2692e5b7b433629550e25356c5accfcc0a6861c996ba8ddc65aa871b67118f72769227b
+  metadata.gz: 66b22834a187504875f27de487f1e994d2e8f7bf7e27a4da4a8ef7e00aae67005db73b05c0e81383947170016ab9c00c665d6955489babc511ec0845eedceb33
+  data.tar.gz: f802830a593f06048d37dd710b7e178e3efe7c2dfd1f0ab6e53f6e29d28a70f026402a6e2675ac13fb65ef608f41e2f812ddae0e65ffe8bf83bbcdc5730495db

data/README.md

@@ -17,7 +17,16 @@ MySQL input plugins for Embulk loads records from MySQL.
 - **table**: destination name (string, required)
 - **select**: comma-separated list of columns to select (string, default: "*")
 - **where**: WHERE condition to filter the rows (string, default: no-condition)
-- **fetch_rows**: number of rows to fetch one time (used for java.sql.Statement#setFetchSize) (integer, default: 10000)
+- **fetch_rows**: number of rows to fetch one time (integer, default: 10000)
+  - If this value is set to > 1:
+    - It uses a server-side prepared statement and fetches rows by chunks.
+    - Internally, `useCursorFetch=true` is enabled and `java.sql.Statement.setFetchSize` is set to the configured value.
+  - If this value is set to 1:
+    - It uses a client-side built statement and fetches rows one by one.
+    - Internally, `useCursorFetch=false` is used and `java.sql.Statement.setFetchSize` is set to Integer.MIN_VALUE.
+  - If this value is set to -1:
+    - It uses a client-side built statement and fetches all rows at once. This may cause OutOfMemoryError.
+    - Internally, `useCursorFetch=false` is used and `java.sql.Statement.setFetchSize` is not set.
 - **options**: extra JDBC properties (hash, default: {})
 
 ## Example

data/src/main/java/org/embulk/input/MySQLInputPlugin.java

@@ -6,6 +6,7 @@ import java.sql.Driver;
 import java.sql.SQLException;
 import com.google.common.base.Throwables;
 import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
 import org.embulk.input.mysql.MySQLInputConnection;
 
@@ -53,6 +54,15 @@ public class MySQLInputPlugin
         //    break;
         //}
 
+        if (task.getFetchRows() == 1) {
+            logger.info("Fetch size is 1. Fetching rows one by one.");
+        } else if (task.getFetchRows() <= 0) {
+            logger.info("Fetch size is set to -1. Fetching all rows at once.");
+        } else {
+            logger.info("Fetch size is {}. Using server-side prepared statement.", task.getFetchRows());
+            props.setProperty("useCursorFetch", "true");
+        }
+
         props.putAll(task.getOptions());
 
         Driver driver;

data/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java

@@ -3,6 +3,7 @@ package org.embulk.input.mysql;
 import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
+import java.sql.ResultSet;
 import org.embulk.input.jdbc.JdbcInputConnection;
 
 public class MySQLInputConnection
@@ -13,4 +14,22 @@ public class MySQLInputConnection
     {
         super(connection, null);
     }
+
+    @Override
+    protected BatchSelect newBatchSelect(String select, int fetchRows) throws SQLException
+    {
+        logger.info("SQL: " + select);
+        PreparedStatement stmt = connection.prepareStatement(select, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);  // TYPE_FORWARD_ONLY and CONCUR_READ_ONLY are default
+        if (fetchRows == 1) {
+            // See MySQLInputPlugin.newConnection doesn't set useCursorFetch=true when fetchRows=1
+            // MySQL Connector/J keeps the connection opened and process rows one by one with Integer.MIN_VALUE.
+            stmt.setFetchSize(Integer.MIN_VALUE);
+        } else if (fetchRows <= 0) {
+            // uses the default behavior. MySQL Connector/J fetches the all rows in memory.
+        } else {
+            // useCursorFetch=true is enabled. MySQL creates temporary table and uses multiple select statements to fetch rows.
+            stmt.setFetchSize(fetchRows);
+        }
+        return new SingleSelect(stmt);
+    }
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mysql
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - FURUHASHI Sadayuki
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-02-18 00:00:00.000000000 Z
+date: 2015-02-19 00:00:00.000000000 Z
 dependencies: []
 description: JDBC input plugin is an Embulk plugin that loads records from JDBC so that any output plugins can receive the records. Search the output plugins by "embulk-output" keyword.
 email:
@@ -22,8 +22,8 @@ files:
 - lib/embulk/input/mysql.rb
 - src/main/java/org/embulk/input/MySQLInputPlugin.java
 - src/main/java/org/embulk/input/mysql/MySQLInputConnection.java
-- classpath/embulk-input-jdbc-0.2.1.jar
-- classpath/embulk-input-mysql-0.2.1.jar
+- classpath/embulk-input-jdbc-0.2.2.jar
+- classpath/embulk-input-mysql-0.2.2.jar
 - classpath/mysql-connector-java-5.1.34.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses: