embulk-input-mysql 0.8.0 → 0.8.1

data/src/test/java/org/embulk/input/mysql/IncrementalTest.java

@@ -0,0 +1,107 @@
+package org.embulk.input.mysql;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.input.MySQLInputPlugin;
+import org.embulk.spi.InputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.embulk.test.TestingEmbulk.RunResult;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.nio.file.Path;
+
+import static org.embulk.input.mysql.MySQLTests.execute;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+public class IncrementalTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/input/mysql/test/expect/incremental/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(InputPlugin.class, "mysql", MySQLInputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+    }
+
+    @Test
+    public void testInt() throws Exception
+    {
+        // setup first rows
+        execute(readResource("int/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "int/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("int/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "int/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("int/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "int/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("int/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "int/expected_2.diff")));
+    }
+
+    @Test
+    public void testDateTime() throws Exception
+    {
+        // setup first rows
+        execute(readResource("dt/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "dt/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("dt/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "dt/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("dt/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "dt/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("dt/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "dt/expected_2.diff")));
+    }
+
+    @Test
+    public void testTimestamp() throws Exception
+    {
+        // setup first rows
+        execute(readResource("ts/setup.sql"));
+
+        Path out1 = embulk.createTempFile("csv");
+        RunResult result1 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "ts/config_1.yml")), out1);
+        assertThat(readSortedFile(out1), is(readResource("ts/expected_1.csv")));
+        assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "ts/expected_1.diff")));
+
+        // insert more rows
+        execute(readResource("ts/insert_more.sql"));
+
+        Path out2 = embulk.createTempFile("csv");
+        RunResult result2 = embulk.runInput(baseConfig.merge(loadYamlResource(embulk, "ts/config_2.yml")), out2);
+        assertThat(readSortedFile(out2), is(readResource("ts/expected_2.csv")));
+        assertThat(result2.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "ts/expected_2.diff")));
+    }
+}

data/src/test/java/org/embulk/input/mysql/MySQLTests.java

@@ -0,0 +1,55 @@
+package org.embulk.input.mysql;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkTests;
+
+import java.io.IOException;
+
+import static java.util.Locale.ENGLISH;
+
+public class MySQLTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_INPUT_MYSQL_TEST_CONFIG");
+    }
+
+    public static void execute(String sql)
+    {
+        ConfigSource config = baseConfig();
+
+        ImmutableList.Builder<String> args = ImmutableList.builder();
+        args.add("mysql")
+                .add("-u")
+                .add(config.get(String.class, "user"));
+        if (!config.get(String.class, "password").isEmpty()) {
+            args.add("-p" + config.get(String.class, "password"));
+        }
+        args
+                .add("-h")
+                .add(config.get(String.class, "host"))
+                .add("-P")
+                .add(config.get(String.class, "port", "3306"))
+                .add(config.get(String.class, "database"))
+                .add("-e")
+                .add(sql);
+
+        ProcessBuilder pb = new ProcessBuilder(args.build());
+        pb.redirectErrorStream(true);
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                    "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/setup.sql

@@ -0,0 +1,65 @@
+drop table if exists test1;
+
+create table test1 (
+    id  char(2),
+    c1  tinyint,
+    c2  smallint,
+    c3  int,
+    c4  bigint,
+    c5  float,
+    c6  double,
+    c7  decimal(4,0),
+    c8  decimal(20,2),
+    c9  char(4),
+    c10 varchar(4),
+    c11 date,
+    c12 datetime,
+    c13 timestamp,
+    c14 time,
+    c15 datetime(6),
+    primary key(id)
+);
+
+insert into test1 values(
+    '10',
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    null,
+    '2015-06-04 23:45:06',
+    null,
+    null
+);
+
+insert into test1 values(
+    '11',
+    99,
+    9999,
+    -99999999,
+    -9999999999999999,
+    1.2345,
+    1.234567890123,
+    -1234,
+    123456789012345678.12,
+    '5678',
+    'xy',
+    '2015-06-04',
+    '2015-06-04 12:34:56',
+    '2015-06-04 23:45:06',
+    '08:04:02',
+    '2015-06-04 01:02:03.123456'
+);
+
+drop table if exists test2;
+
+create table test2 (c1 bigint unsigned);
+
+insert into test2 values(18446744073709551615);

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_config.yml

@@ -0,0 +1,13 @@
+table: test1
+order_by: 'id'
+column_options:
+  c1: {type: boolean}
+  c2: {type: boolean}
+  c3: {type: boolean}
+  c4: {type: boolean}
+  c5: {type: boolean}
+  c6: {type: boolean}
+  c7: {type: boolean}
+  c8: {type: boolean}
+  c9: {type: boolean}
+  c10: {type: boolean}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,true,true,false,false,true,true,false,true,,,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_boolean_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_config.yml

@@ -0,0 +1,2 @@
+table: test1
+order_by: 'id'

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_config.yml

@@ -0,0 +1,13 @@
+table: test1
+order_by: 'id'
+column_options:
+  c1: {type: double}
+  c2: {type: double}
+  c3: {type: double}
+  c4: {type: double}
+  c5: {type: double}
+  c6: {type: double}
+  c7: {type: double}
+  c8: {type: double}
+  c9: {type: double}
+  c10: {type: double}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99.0,9999.0,-9.9999999E7,-1.0E16,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678.0,,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_double_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_config.yml

@@ -0,0 +1,13 @@
+table: test1
+order_by: 'id'
+column_options:
+  c1: {type: long}
+  c2: {type: long}
+  c3: {type: long}
+  c4: {type: long}
+  c5: {type: long}
+  c6: {type: long}
+  c7: {type: long}
+  c8: {type: long}
+  c9: {type: long}
+  c10: {type: long}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99,9999,-99999999,-9999999999999999,1,1,-1234,123456789012345678,5678,,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_long_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_config.yml

@@ -0,0 +1,13 @@
+table: test1
+order_by: 'id'
+column_options:
+  c1: {type: string}
+  c2: {type: string}
+  c3: {type: string}
+  c4: {type: string}
+  c5: {type: string}
+  c6: {type: string}
+  c7: {type: string}
+  c8: {type: string}
+  c9: {type: string}
+  c10: {type: string}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99,9999,-99999999,-9999999999999999,1.2345,1.234567890123,-1234,123456789012345678.12,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_string_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_config.yml

@@ -0,0 +1,13 @@
+table: test1
+order_by: 'id'
+column_options:
+  #c11: {type: timestamp, timestamp_format: '%Y/%m/%d'}
+  #c12: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S'}
+  #c13: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
+  #c14: {type: timestamp, timestamp_format: '%H-%M-%S', timezone: '+0900'}
+  #c15: {type: timestamp, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}
+  c11: {type: timestamp}
+  c12: {type: timestamp}
+  c13: {type: timestamp}
+  c14: {type: timestamp}
+  c15: {type: timestamp}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp1_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_config.yml

@@ -0,0 +1,8 @@
+table: test1
+order_by: 'id'
+column_options:
+  c11: {type: string, timestamp_format: '%Y/%m/%d'}
+  c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
+  c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S', timezone: '+0900'}
+  c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0900'}
+  c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015/06/05 05:45:06,,
+11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/03,2015/06/04 09:34:56,2015/06/05 05:45:06,15-04-02,2015/06/03 22:02:03.123456

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp2_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_config.yml

@@ -0,0 +1,9 @@
+table: test1
+order_by: 'id'
+default_timezone: '+0900'
+column_options:
+  c11: {type: string, timestamp_format: '%Y/%m/%d'}
+  c12: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
+  c13: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S'}
+  c14: {type: string, timestamp_format: '%H-%M-%S', timezone: '+0300'}
+  c15: {type: string, timestamp_format: '%Y/%m/%d %H:%M:%S.%6N'}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015/06/05 05:45:06,,
+11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015/06/04,2015/06/04 18:34:56,2015/06/05 05:45:06,09-04-02,2015/06/04 07:02:03.123456

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_timestamp3_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_config.yml

@@ -0,0 +1,3 @@
+table: test1
+column_options:
+  c1: {value_type: decimal}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99.0,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_decimal_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_config.yml

@@ -0,0 +1,3 @@
+table: test1
+column_options:
+  c1: {value_type: string}

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_expected.csv

@@ -0,0 +1,2 @@
+10,,,,,,,,,,,,,2015-06-04 20:45:06.000000 +0000,,
+11,99,9999,-99999999,-9999999999999999,1.2345000505447388,1.234567890123,-1234.0,1.2345678901234568E17,5678,xy,2015-06-03 21:00:00.000000 +0000,2015-06-04 09:34:56.000000 +0000,2015-06-04 20:45:06.000000 +0000,1970-01-01 06:04:02.000000 +0000,2015-06-03 22:02:03.123456 +0000

data/src/test/resources/org/embulk/input/mysql/test/expect/basic/test_valuetype_string_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/config_1.yml

@@ -0,0 +1,5 @@
+table: dt_load
+default_time_zone: +0300 # should be ignored
+options: {useLegacyDatetimeCode: false}
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/config_2.yml

@@ -0,0 +1,5 @@
+table: dt_load
+options: {useLegacyDatetimeCode: false}
+last_record: ['2016-11-02T04:00:05.333003']
+incremental: true
+incremental_columns: [time]

data/src/test/resources/org/embulk/input/mysql/test/expect/incremental/dt/expected_1.csv

@@ -0,0 +1,7 @@
+2016-11-02 01:00:01.000000 +0000,first
+2016-11-02 02:00:02.000000 +0000,first
+2016-11-02 03:00:03.000000 +0000,first
+2016-11-02 04:00:04.000000 +0000,first
+2016-11-02 04:00:05.111001 +0000,first
+2016-11-02 04:00:05.222002 +0000,first
+2016-11-02 04:00:05.333003 +0000,first