embulk-input-mysql 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2144fd45cabb9eb049b87ef7307119912982544d
-  data.tar.gz: 18b30e932ba4ae4db386f1ead1d6b93780c94830
+  metadata.gz: 4797ea2ce2deddb157e0d4804267a776bd163420
+  data.tar.gz: 13494a5d27aabf8cbea454ed97c0b25e4ce46077
 SHA512:
-  metadata.gz: 82f1d70e1d75acfd47b71e1e679a4eb085dd4fbc22ef562807a1045b697f77f2980e64663807503a4b2fb55d93365b594ecd0b1f7492e597c5262bb3edce47b2
-  data.tar.gz: 3ee4d8e3dd1570c230fdcd0ac684122998d728df75c143b7ac2b70355b2d91268424d675dcde52dfc46baa358e1f4d3a3ecb68959dd1e3561f642b776551d909
+  metadata.gz: dc654e504b7bceee5836fef9f3d57ec73a9b47ae39fc8f332a63866c6be097babb2f953f6a067a54055c90af4b6bc09bd9c2a7ea8e69d7b05504cb4047fb8739
+  data.tar.gz: aaa3cefa4f48abaa0e7bafe1bae8db92c29d8787c8a561ce65b13c70d2bced39a8ce135a389b48d85c9332c9c75ed459cc3aa47f4a85aa21b6bfbc3dd4df5215

data/README.md

@@ -150,3 +150,12 @@ in:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+```
+$ cp ci/travis_mysql.yml ci/mysql.yml  # edit this file if necessary
+$ EMBULK_INPUT_MYSQL_TEST_CONFIG=`pwd`/ci/mysql.yml ./gradlew :embulk-input-mysql:check --info
+```
+
+This test data are expected by using 'UTC' as MySQL server's timezone. On the other hand, unit tests use 'Europe/Helsinki' as jdbc driver's session timezone.

data/build.gradle

@@ -3,6 +3,5 @@ dependencies {
 
     compile 'mysql:mysql-connector-java:5.1.34'
 
-    testCompile 'org.embulk:embulk-standards:0.8.8'
-    testCompile project(':embulk-input-jdbc').sourceSets.test.output
+    testCompile 'org.embulk:embulk-standards:0.8.15'
 }

data/src/main/java/org/embulk/input/MySQLInputPlugin.java

@@ -1,13 +1,22 @@
 package org.embulk.input;
 
+import java.io.IOException;
+import java.lang.reflect.Field;
 import java.util.Properties;
 import java.sql.Connection;
 import java.sql.Driver;
 import java.sql.SQLException;
+
+import com.google.common.base.Throwables;
+import com.mysql.jdbc.TimeUtil;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.input.jdbc.AbstractJdbcInputPlugin;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
 import org.embulk.input.mysql.MySQLInputConnection;
+import org.embulk.input.mysql.getter.MySQLColumnGetterFactory;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
 
 public class MySQLInputPlugin
         extends AbstractJdbcInputPlugin
@@ -91,6 +100,9 @@ public class MySQLInputPlugin
 
         props.putAll(t.getOptions());
 
+        // load timezone mappings
+        loadTimeZoneMappings();
+
         Driver driver;
         try {
             driver = new com.mysql.jdbc.Driver();  // new com.mysql.jdbc.Driver throws SQLException
@@ -109,4 +121,46 @@ public class MySQLInputPlugin
             }
         }
     }
+
+    @Override
+    protected ColumnGetterFactory newColumnGetterFactory(PageBuilder pageBuilder, DateTimeZone dateTimeZone)
+    {
+        return new MySQLColumnGetterFactory(pageBuilder, dateTimeZone);
+    }
+
+    private void loadTimeZoneMappings()
+    {
+        // Here initializes com.mysql.jdbc.TimeUtil.timeZoneMappings static field by calling
+        // static timeZoneMappings method using reflection.
+        // The field is usually initialized when Driver#connect method is called. But the field
+        // initialization fails when a) useLegacyDatetimeCode=false is set AND b) mysql server's
+        // default_time_zone is not SYSTEM (default). According to the stacktrace, that's because
+        // the com.mysql.jdbc.TimeUtil.loadTimeZoneMappings can't find TimeZoneMapping.properties
+        // from the classloader. It seems like a bug of JDBC Driver where it should use the class loader
+        // that loaded com.mysql.jdbc.TimeUtil class rather than system class loader to read the
+        // property file because the file should be in the same classpath with the class.
+        // Here implements a workaround as as workaround.
+        Field f = null;
+        try {
+            f = TimeUtil.class.getDeclaredField("timeZoneMappings");
+            f.setAccessible(true);
+
+            Properties timeZoneMappings = (Properties) f.get(null);
+            if (timeZoneMappings == null) {
+                timeZoneMappings = new Properties();
+                synchronized (TimeUtil.class) {
+                    timeZoneMappings.load(this.getClass().getResourceAsStream("/com/mysql/jdbc/TimeZoneMapping.properties"));
+                }
+                f.set(null, timeZoneMappings);
+            }
+        }
+        catch (IllegalAccessException | NoSuchFieldException | IOException e) {
+            throw Throwables.propagate(e);
+        }
+        finally {
+            if (f != null) {
+                f.setAccessible(false);
+            }
+        }
+    }
 }

data/src/main/java/org/embulk/input/mysql/MySQLInputConnection.java

@@ -5,6 +5,10 @@ import java.sql.Connection;
 import java.sql.PreparedStatement;
 import java.sql.SQLException;
 import java.sql.ResultSet;
+import java.util.TimeZone;
+
+import com.mysql.jdbc.ConnectionImpl;
+import com.mysql.jdbc.ConnectionProperties;
 import org.embulk.input.jdbc.JdbcInputConnection;
 import org.embulk.input.jdbc.JdbcLiteral;
 import org.embulk.input.jdbc.getter.ColumnGetter;
@@ -45,4 +49,14 @@ public class MySQLInputConnection
         // Because socketTimeout is set in Connection, don't need to set quertyTimeout.
         return new SingleSelect(stmt);
     }
+
+    public boolean getUseLegacyDatetimeCode()
+    {
+        return ((ConnectionProperties) connection).getUseLegacyDatetimeCode();
+    }
+
+    public TimeZone getServerTimezoneTZ()
+    {
+        return ((ConnectionImpl) connection).getServerTimezoneTZ();
+    }
 }

data/src/main/java/org/embulk/input/mysql/getter/AbstractMySQLTimestampIncrementalHandler.java

@@ -0,0 +1,75 @@
+package org.embulk.input.mysql.getter;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import org.embulk.input.jdbc.getter.AbstractIncrementalHandler;
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.time.TimestampFormatter;
+import org.embulk.spi.time.TimestampFormatter.FormatterTask;
+import org.embulk.spi.time.TimestampParser;
+import org.embulk.spi.time.TimestampParser.ParserTask;
+import org.joda.time.DateTimeZone;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+
+public abstract class AbstractMySQLTimestampIncrementalHandler
+        extends AbstractIncrementalHandler
+{
+    protected final DateTimeZone sessionTimeZone;
+    protected long epochSecond;
+    protected int nano;
+
+    public AbstractMySQLTimestampIncrementalHandler(DateTimeZone sessionTimeZone, ColumnGetter next)
+    {
+        super(next);
+        this.sessionTimeZone = sessionTimeZone;
+    }
+
+    @Override
+    public void getAndSet(ResultSet from, int fromIndex, Column toColumn)
+            throws SQLException
+    {
+        Timestamp timestamp = from.getTimestamp(fromIndex);
+        if (timestamp != null) {
+            epochSecond = timestamp.getTime() / 1000;
+            nano = timestamp.getNanos();
+        }
+
+        super.getAndSet(from, fromIndex, toColumn);
+    }
+
+    @Override
+    public JsonNode encodeToJson()
+    {
+        FormatterTask task = Exec.newConfigSource()
+            .set("timezone", "UTC")
+            .loadConfig(FormatterTask.class);
+        TimestampFormatter formatter = new TimestampFormatter(getTimestampFormat(), task);
+        String text = formatter.format(utcTimestampFromSessionTime(epochSecond, nano));
+        return jsonNodeFactory.textNode(text);
+    }
+
+    protected abstract String getTimestampFormat();
+
+    protected abstract org.embulk.spi.time.Timestamp utcTimestampFromSessionTime(long epochSecond, int nano);
+
+    @Override
+    public void decodeFromJsonTo(PreparedStatement toStatement, int toIndex, JsonNode fromValue)
+            throws SQLException
+    {
+        ParserTask task = Exec.newConfigSource()
+            .set("default_timezone", "UTC")
+            .loadConfig(ParserTask.class);
+        TimestampParser parser = new TimestampParser(getTimestampPattern(), task);
+        org.embulk.spi.time.Timestamp epoch = parser.parse(fromValue.asText());
+        toStatement.setTimestamp(toIndex, utcTimestampToSessionTime(epoch));
+    }
+
+    protected abstract String getTimestampPattern();
+
+    protected abstract Timestamp utcTimestampToSessionTime(org.embulk.spi.time.Timestamp ts);
+}

data/src/main/java/org/embulk/input/mysql/getter/MySQLColumnGetterFactory.java

@@ -0,0 +1,61 @@
+package org.embulk.input.mysql.getter;
+
+import com.google.common.base.Optional;
+import org.embulk.config.ConfigException;
+import org.embulk.input.jdbc.AbstractJdbcInputPlugin.PluginTask;
+import org.embulk.input.jdbc.JdbcColumn;
+import org.embulk.input.jdbc.JdbcColumnOption;
+import org.embulk.input.jdbc.JdbcInputConnection;
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.embulk.input.jdbc.getter.ColumnGetterFactory;
+import org.embulk.input.mysql.MySQLInputConnection;
+import org.embulk.spi.PageBuilder;
+import org.joda.time.DateTimeZone;
+
+import java.util.TimeZone;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+public class MySQLColumnGetterFactory
+        extends ColumnGetterFactory
+{
+    public MySQLColumnGetterFactory(PageBuilder to, DateTimeZone defaultTimeZone)
+    {
+        super(to, defaultTimeZone);
+    }
+
+    @Override
+    public ColumnGetter newColumnGetter(JdbcInputConnection con, PluginTask task, JdbcColumn column, JdbcColumnOption option)
+    {
+        ColumnGetter getter = super.newColumnGetter(con, task, column, option);
+
+        switch (column.getTypeName()) {
+        case "DATETIME":
+        case "TIMESTAMP":
+            int index = task.getQuerySchema().findColumn(column.getName()).get();
+            if (!task.getIncremental() || !task.getIncrementalColumnIndexes().contains(index)) {
+                return getter;
+            }
+
+            // incremental loading
+            MySQLInputConnection mysqlInputConnection = (MySQLInputConnection) con;
+            // Users cannot use DATETIME or TIMESTAMP typed columns as incremental_columns: if 'useLegacyDatetimeCode=true'.
+            // That might be acceptable since mysql-connector-java v6.x will turn off, by default.
+            if (mysqlInputConnection.getUseLegacyDatetimeCode()) {
+                throw new ConfigException("Must use 'useLegacyDatetimeCode=false' if 'DATETIME' or 'TIMESTAMP' typed columns are used as incremental_columns:");
+            }
+
+            TimeZone timeZone = mysqlInputConnection.getServerTimezoneTZ();
+            // Joda-Time's timezone mapping is probably not compatible with java.util.TimeZone if null is returned.
+            DateTimeZone sessionTimeZone = checkNotNull(DateTimeZone.forTimeZone(timeZone));
+            if (column.getTypeName().equals("DATETIME")) {
+                return new MySQLDateTimeTimestampIncrementalHandler(sessionTimeZone, getter);
+            }
+            else { // TIMESTAMP
+                return new MySQLTimestampTimestampIncrementalHandler(sessionTimeZone, getter);
+            }
+        default:
+            return getter;
+        }
+    }
+}

data/src/main/java/org/embulk/input/mysql/getter/MySQLDateTimeTimestampIncrementalHandler.java

@@ -0,0 +1,45 @@
+package org.embulk.input.mysql.getter;
+
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.joda.time.DateTimeZone;
+
+import java.sql.Timestamp;
+
+public class MySQLDateTimeTimestampIncrementalHandler
+        extends AbstractMySQLTimestampIncrementalHandler
+{
+    public MySQLDateTimeTimestampIncrementalHandler(DateTimeZone sessionTimeZone, ColumnGetter next)
+    {
+        super(sessionTimeZone, next);
+    }
+
+    @Override
+    public String getTimestampFormat()
+    {
+        return "%Y-%m-%dT%H:%M:%S.%6N";
+    }
+
+    @Override
+    public org.embulk.spi.time.Timestamp utcTimestampFromSessionTime(long epochSecond, int nano)
+    {
+        // this Timestamp value is already converted by session time_zone.
+        long reconverted = sessionTimeZone.convertUTCToLocal(epochSecond * 1000) / 1000; // reconvert from session time_zone to UTC
+        return org.embulk.spi.time.Timestamp.ofEpochSecond(reconverted, nano);
+    }
+
+    @Override
+    public String getTimestampPattern()
+    {
+        return "%Y-%m-%dT%H:%M:%S.%N";
+    }
+
+    @Override
+    public Timestamp utcTimestampToSessionTime(org.embulk.spi.time.Timestamp from)
+    {
+        // reconvert from UTC to session time_zone
+        long reconverted = sessionTimeZone.convertLocalToUTC(from.getEpochSecond() * 1000, false);
+        Timestamp sqlTimestamp = new Timestamp(reconverted);
+        sqlTimestamp.setNanos(from.getNano());
+        return sqlTimestamp;
+    }
+}

data/src/main/java/org/embulk/input/mysql/getter/MySQLTimestampTimestampIncrementalHandler.java

@@ -0,0 +1,42 @@
+package org.embulk.input.mysql.getter;
+
+import org.embulk.input.jdbc.getter.ColumnGetter;
+import org.joda.time.DateTimeZone;
+
+import java.sql.Timestamp;
+
+public class MySQLTimestampTimestampIncrementalHandler
+        extends AbstractMySQLTimestampIncrementalHandler
+{
+    public MySQLTimestampTimestampIncrementalHandler(DateTimeZone sessionTimeZone, ColumnGetter next)
+    {
+        super(sessionTimeZone, next);
+    }
+
+    @Override
+    public String getTimestampFormat()
+    {
+        return "%Y-%m-%dT%H:%M:%S.%6NZ";
+    }
+
+    @Override
+    public org.embulk.spi.time.Timestamp utcTimestampFromSessionTime(long epochSecond, int nano)
+    {
+        long sec = sessionTimeZone.convertLocalToUTC(epochSecond * 1000, false) / 1000;
+        return org.embulk.spi.time.Timestamp.ofEpochSecond(sec, nano);
+    }
+
+    @Override
+    public String getTimestampPattern()
+    {
+        return "%Y-%m-%dT%H:%M:%S.%N%z";
+    }
+
+    @Override
+    public Timestamp utcTimestampToSessionTime(org.embulk.spi.time.Timestamp ts)
+    {
+        Timestamp sqlTimestamp = new Timestamp(ts.getEpochSecond() * 1000);
+        sqlTimestamp.setNanos(ts.getNano());
+        return sqlTimestamp;
+    }
+}

data/src/test/java/org/embulk/input/mysql/BasicTest.java

@@ -0,0 +1,137 @@
+package org.embulk.input.mysql;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.MySQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.embulk.input.mysql.MySQLTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class BasicTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/mysql/test/expect/basic/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "mysql", MySQLInputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void test() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testString() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_string_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_string_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_string_expected.diff")));
+    }
+
+    @Test
+    public void testBoolean() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_boolean_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_boolean_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_boolean_expected.diff")));
+    }
+
+    @Test
+    public void testLong() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_long_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_long_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_long_expected.diff")));
+    }
+
+    @Test
+    public void testDouble() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_double_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_double_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_double_expected.diff")));
+    }
+
+    @Test
+    public void testTimestamp1() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_timestamp1_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_timestamp1_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_timestamp1_expected.diff")));
+    }
+
+    @Test
+    public void testTimestamp2() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_timestamp2_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_timestamp2_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_timestamp2_expected.diff")));
+    }
+
+    @Test
+    public void testTimestamp3() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_timestamp3_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_timestamp3_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_timestamp3_expected.diff")));
+    }
+
+    @Test
+    public void testValueTypeString() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_valuetype_string_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_valuetype_string_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_valuetype_string_expected.diff")));
+    }
+
+    @Test
+    public void testValueTypeDecimal() throws Exception
+    {
+        Path out1 = embulk.createTempFile("csv");
+        TestingEmbulk.RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "test_valuetype_decimal_config.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("test_valuetype_decimal_expected.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_valuetype_decimal_expected.diff")));
+    }
+}