embulk-input-mysql 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd90341b77cceca9532bf385439c2437f8caf517
-  data.tar.gz: dd81a39ddc867c7e0c577096792d2178b237dfb9
+  metadata.gz: 926a4a94fc9d68597a855726bff1ee7f0aa9642f
+  data.tar.gz: 65fb022951c42b95037d989f0e8f0229a500ac60
 SHA512:
-  metadata.gz: ddd6c09b93413f5bcc36120ba1841193c4f3e662f9f2e8d1652f75c3ebc609113c7c6a144386ccb3ef8956c5eb383d9addfe7f3380f60eabf04b9ae40b6db47a
-  data.tar.gz: b7669585a6c96283717542589ed978b53b17c905c7d11cb41d63453e07cfe7783de1a8e73731226c830582d48d39d2393fea1d62e46e1c5f09b2cae8d4bfc2c0
+  metadata.gz: ba17b951e574ce78a5605a0ae2c1970b1de516044fd7d627fb6e55c7b62ae55f084a9b21ac5033f5c684534093b3f6352b25e52c10fd7dcab7064471ca733eed
+  data.tar.gz: 4363441acac9ce2d250d85f1a6bf425d961e07b9276d2190c98f0d2b0a89f3c8769d5a963401e794e49414879199b33333faa7a681eac1d7226bb9f6813d1577

data/README.md

@@ -30,6 +30,8 @@ MySQL input plugins for Embulk loads records from MySQL.
   - If this value is set to -1:
     - It uses a client-side built statement and fetches all rows at once. This may cause OutOfMemoryError.
     - Internally, `useCursorFetch=false` is used and `java.sql.Statement.setFetchSize` is not set.
+- **connect_timeout**: timeout for socket connect. 0 means no timeout. (integer (seconds), default: 300)
+- **socket_timeout**: timeout on network socket operations. 0 means no timeout. (integer (seconds), default: 1800)
 - **options**: extra JDBC properties (hash, default: {})
 - **default_timezone**: If the sql type of a column is `date`/`time`/`datetime` and the embulk type is `string`, column values are formatted int this default_timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.

data/build.gradle

@@ -3,6 +3,6 @@ dependencies {
 
     compile 'mysql:mysql-connector-java:5.1.34'
 
-    testCompile 'org.embulk:embulk-standards:0.6.16'
+    testCompile 'org.embulk:embulk-standards:0.7.4'
     testCompile project(':embulk-input-jdbc').sourceSets.test.output
 }

data/src/main/java/org/embulk/input/MySQLInputPlugin.java

@@ -4,7 +4,6 @@ import java.util.Properties;
 import java.sql.Connection;
 import java.sql.Driver;
 import java.sql.SQLException;
-import com.google.common.base.Throwables;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
@@ -58,8 +57,8 @@ public class MySQLInputPlugin
 
         props.setProperty("useCompression", "true");
 
-        props.setProperty("connectTimeout", "300000"); // milliseconds
-        props.setProperty("socketTimeout", "1800000"); // smillieconds
+        props.setProperty("connectTimeout", String.valueOf(t.getConnectTimeout() * 1000)); // milliseconds
+        props.setProperty("socketTimeout", String.valueOf(t.getSocketTimeout() * 1000)); // milliseconds
 
         // Enable keepalive based on tcp_keepalive_time, tcp_keepalive_intvl and tcp_keepalive_probes kernel parameters.
         // Socket options TCP_KEEPCNT, TCP_KEEPIDLE, and TCP_KEEPINTVL are not configurable.

data/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java

@@ -16,7 +16,7 @@ public class MySQLInputConnection
     }
 
     @Override
-    protected BatchSelect newBatchSelect(String select, int fetchRows) throws SQLException
+    protected BatchSelect newBatchSelect(String select, int fetchRows, int queryTimeout) throws SQLException
     {
         logger.info("SQL: " + select);
         PreparedStatement stmt = connection.prepareStatement(select, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);  // TYPE_FORWARD_ONLY and CONCUR_READ_ONLY are default
@@ -30,6 +30,7 @@ public class MySQLInputConnection
             // useCursorFetch=true is enabled. MySQL creates temporary table and uses multiple select statements to fetch rows.
             stmt.setFetchSize(fetchRows);
         }
+        // Because socketTimeout is set in Connection, don't need to set quertyTimeout.
         return new SingleSelect(stmt);
     }
 }

data/src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java

@@ -2,19 +2,14 @@ package org.embulk.input.mysql;
 
 import java.io.File;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.List;
 
-import org.embulk.EmbulkService;
-import org.embulk.config.ConfigLoader;
+import org.embulk.EmbulkEmbed;
+import org.embulk.EmbulkEmbed.Bootstrap;
 import org.embulk.config.ConfigSource;
-import org.embulk.exec.BulkLoader;
-import org.embulk.exec.ExecutionResult;
 import org.embulk.plugin.InjectedPluginSource;
-import org.embulk.spi.ExecSession;
 
 import com.google.inject.Binder;
-import com.google.inject.Injector;
 import com.google.inject.Module;
 
 public class EmbulkPluginTester
@@ -37,7 +32,7 @@ public class EmbulkPluginTester
 
     private final List<PluginDefinition> plugins = new ArrayList<PluginDefinition>();
 
-
+    private EmbulkEmbed embulk;
 
     public EmbulkPluginTester()
     {
@@ -55,30 +50,29 @@ public class EmbulkPluginTester
 
     public void run(String ymlPath) throws Exception
     {
-        EmbulkService service = new EmbulkService(new EmptyConfigSource()) {
-            @Override
-            protected Iterable<? extends Module> getAdditionalModules(ConfigSource systemConfig)
+        if (embulk == null) {
+            Bootstrap bootstrap = new EmbulkEmbed.Bootstrap();
+            bootstrap.addModules(new Module()
             {
-                return Arrays.asList(new Module()
+                @Override
+                public void configure(Binder binder)
                 {
-                    @Override
-                    public void configure(Binder binder)
-                    {
-                        for (PluginDefinition plugin : plugins) {
-                            InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
-                        }
+                    for (PluginDefinition plugin : plugins) {
+                        InjectedPluginSource.registerPluginTo(binder, plugin.iface, plugin.name, plugin.impl);
                     }
-                });
-            }
-        };
-        Injector injector = service.getInjector();
-        ConfigSource config = injector.getInstance(ConfigLoader.class).fromYamlFile(new File(ymlPath));
-        ExecSession session = new ExecSession(injector, config);
-        try {
-            BulkLoader loader = injector.getInstance(BulkLoader.class);
-            ExecutionResult result = loader.run(session, config);
-        } finally {
-            session.cleanup();
+                }
+            });
+            embulk = bootstrap.initializeCloseable();
+        }
+
+        ConfigSource config = embulk.newConfigLoader().fromYamlFile(new File(ymlPath));
+        embulk.run(config);
+    }
+
+    public void destroy() {
+        if (embulk != null) {
+            embulk.destroy();
+            embulk = null;
         }
     }
 

data/src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java

@@ -16,6 +16,7 @@ import java.util.List;
 
 import org.embulk.input.MySQLInputPlugin;
 import org.embulk.spi.InputPlugin;
+import org.junit.AfterClass;
 import org.junit.BeforeClass;
 import org.junit.Test;
 
@@ -24,6 +25,7 @@ import static org.junit.Assert.assertEquals;
 public class MySQLInputPluginTest
 {
     private static boolean prepared = false;
+    private static EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
 
     @BeforeClass
     public static void prepare() throws SQLException
@@ -115,11 +117,16 @@ public class MySQLInputPluginTest
         }
     }
 
+    @AfterClass
+    public static void dispose()
+    {
+        tester.destroy();
+    }
+
     @Test
     public void test() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -133,7 +140,6 @@ public class MySQLInputPluginTest
     public void testString() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-string.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -147,7 +153,6 @@ public class MySQLInputPluginTest
     public void testBoolean() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-boolean.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -161,7 +166,6 @@ public class MySQLInputPluginTest
     public void testLong() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-long.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -175,7 +179,6 @@ public class MySQLInputPluginTest
     public void testDouble() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-double.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -189,7 +192,6 @@ public class MySQLInputPluginTest
     public void testTimestamp1() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-timestamp1.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -203,7 +205,6 @@ public class MySQLInputPluginTest
     public void testTimestamp2() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-timestamp2.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -217,7 +218,6 @@ public class MySQLInputPluginTest
     public void testTimestamp3() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-timestamp3.yml"));
             assertEquals(Arrays.asList(
                     "c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15",
@@ -231,7 +231,6 @@ public class MySQLInputPluginTest
     public void testValueTypeString() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-valuetype-string.yml"));
             assertEquals(Arrays.asList(
                     "c1",
@@ -244,7 +243,6 @@ public class MySQLInputPluginTest
     public void testValueTypeDecimal() throws Exception
     {
         if (prepared) {
-            EmbulkPluginTester tester = new EmbulkPluginTester(InputPlugin.class, "mysql", MySQLInputPlugin.class);
             tester.run(convertPath("/yml/input-valuetype-decimal.yml"));
             assertEquals(Arrays.asList(
                     "c1",

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-mysql
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-07 00:00:00.000000000 Z
+date: 2015-12-08 00:00:00.000000000 Z
 dependencies: []
 description: Selects records from a table.
 email:
@@ -22,9 +22,7 @@ files:
 - lib/embulk/input/mysql.rb
 - src/main/java/org/embulk/input/MySQLInputPlugin.java
 - src/main/java/org/embulk/input/mysql/MySQLInputConnection.java
-- src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java
 - src/test/java/org/embulk/input/mysql/EmbulkPluginTester.java
-- src/test/java/org/embulk/input/mysql/EmptyConfigSource.java
 - src/test/java/org/embulk/input/mysql/MySQLInputPluginTest.java
 - src/test/resources/yml/input-boolean.yml
 - src/test/resources/yml/input-double.yml
@@ -36,8 +34,8 @@ files:
 - src/test/resources/yml/input-valuetype-decimal.yml
 - src/test/resources/yml/input-valuetype-string.yml
 - src/test/resources/yml/input.yml
-- classpath/embulk-input-jdbc-0.6.0.jar
-- classpath/embulk-input-mysql-0.6.0.jar
+- classpath/embulk-input-jdbc-0.6.1.jar
+- classpath/embulk-input-mysql-0.6.1.jar
 - classpath/mysql-connector-java-5.1.34.jar
 homepage: https://github.com/embulk/embulk-input-jdbc
 licenses:

data/src/test/java/org/embulk/input/mysql/ChildFirstClassLoader.java

@@ -1,42 +0,0 @@
-package org.embulk.input.mysql;
-
-import java.net.URL;
-import java.net.URLClassLoader;
-import java.util.List;
-
-public class ChildFirstClassLoader
-        extends URLClassLoader
-{
-    public ChildFirstClassLoader(List<URL> urls, ClassLoader parent)
-    {
-        super(urls.toArray(new URL[urls.size()]), parent);
-    }
-
-    @Override
-    protected Class<?> loadClass(String name, boolean resolve)
-            throws ClassNotFoundException
-    {
-        synchronized (getClassLoadingLock(name)) {
-            Class<?> loadedClass = findLoadedClass(name);
-            if (loadedClass != null) {
-                return resolveClass(loadedClass, resolve);
-            }
-
-            try {
-                return resolveClass(findClass(name), resolve);
-            } catch (ClassNotFoundException ignored) {
-            }
-
-            return super.loadClass(name, resolve);
-        }
-    }
-
-    private Class<?> resolveClass(Class<?> clazz, boolean resolve)
-    {
-        if (resolve) {
-            resolveClass(clazz);
-        }
-        return clazz;
-    }
-
-}

data/src/test/java/org/embulk/input/mysql/EmptyConfigSource.java

@@ -1,105 +0,0 @@
-package org.embulk.input.mysql;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map.Entry;
-
-import org.embulk.config.ConfigSource;
-import org.embulk.config.DataSource;
-
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-public class EmptyConfigSource implements ConfigSource
-{
-
-    @Override
-    public <E> E get(Class<E> type, String attrName)
-    {
-        return null;
-    }
-
-    @Override
-    public <E> E get(Class<E> type, String attrName, E defaultValue)
-    {
-        return defaultValue;
-    }
-
-    @Override
-    public List<String> getAttributeNames()
-    {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public Iterable<Entry<String, JsonNode>> getAttributes()
-    {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public ObjectNode getObjectNode()
-    {
-        return null;
-    }
-
-    @Override
-    public boolean isEmpty()
-    {
-        return true;
-    }
-
-    @Override
-    public ConfigSource deepCopy()
-    {
-        return this;
-    }
-
-    @Override
-    public ConfigSource getNested(String s)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource getNestedOrSetEmpty(String s)
-    {
-        return null;
-    }
-
-    @Override
-    public <T> T loadConfig(Class<T> class1)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource merge(DataSource datasource)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource set(String s, Object obj)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource setAll(DataSource datasource)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource setNested(String s, DataSource datasource)
-    {
-        return null;
-    }
-
-    @Override
-    public ConfigSource remove(String arg0) {
-        return null;
-    }
-
-}