embulk-output-mysql 0.7.10 → 0.7.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 692465de551dae9499e788eedd698511da8446cd
-  data.tar.gz: a35c2e193ed89e1402d08abb9562e55c6a29beb1
+  metadata.gz: 53e6e129b50f732ddc97a6a9b536be38cd7b9093
+  data.tar.gz: 967e7d6b3e1e7927fdf99b317e084734ba91d3b1
 SHA512:
-  metadata.gz: 42d2e2458b4d575eaa38b0e1fbb8b9e4aed52c96c8a8c9ffa6d31bd2f14d0448f3445cdb5d6990f0747dd587060e2e1f55666a248d7bf8bc29681937a25889dd
-  data.tar.gz: cfe4ff6a46f0cddc4001680d98ddc422965a74fd7aadc282944bd6887f49407b023560483cac9893ee39f9849a6ede18f5b52c35d8d60769adcd8ad789d19b45
+  metadata.gz: 3a346fb0bc27a2eb2678ce3a45104546e27e7b5bd35654ffceb36359cd18a8d75d04a17df828746729cedb98337a3aa5e109fa898b445e787411db2416467d92
+  data.tar.gz: a865c64bbe9aa04e8b7a822f1393817a056ca49e6020e8b90a15a55df62354330af2c46edc53de7f6b08c5326ad2886506e3c4831fddc75abe55948a7172ac3b

data/README.md

@@ -120,3 +120,19 @@ out:
 ```
 $ ./gradlew gem
 ```
+
+Running tests:
+
+You need to create 'mysql.yml' as follows.
+```
+type: mysql
+host: localhost
+port: 3306
+database: database
+user: user
+password: pass
+```
+
+```
+$ EMBULK_OUTPUT_MYSQL_TEST_CONFIG=mysql.yml ./gradlew :embulk-output-mysql:check --info
+```

data/build.gradle

@@ -2,6 +2,5 @@ dependencies {
     compile project(':embulk-output-jdbc')
     compile 'mysql:mysql-connector-java:5.1.34'
 
-    testCompile 'org.embulk:embulk-standards:0.8.8'
-    testCompile project(':embulk-output-jdbc').sourceSets.test.output
+    testCompile 'org.embulk:embulk-standards:0.8.22'
 }

data/src/test/java/org/embulk/output/mysql/AfterLoadTest.java

@@ -0,0 +1,136 @@
+package org.embulk.output.mysql;
+
+import static org.embulk.output.mysql.MySQLTests.execute;
+import static org.embulk.output.mysql.MySQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class AfterLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/after_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_replace_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_replace_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testReplaceAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectAfterLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_after_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_after_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/java/org/embulk/output/mysql/BeforeLoadTest.java

@@ -0,0 +1,124 @@
+package org.embulk.output.mysql;
+
+import static org.embulk.output.mysql.MySQLTests.execute;
+import static org.embulk.output.mysql.MySQLTests.selectRecords;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assert.assertThat;
+
+import java.io.File;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.FileSystems;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import com.google.common.io.Resources;
+
+public class BeforeLoadTest
+{
+    private static final String BASIC_RESOURCE_PATH = "org/embulk/output/mysql/test/expect/before_load/";
+
+    private static ConfigSource loadYamlResource(TestingEmbulk embulk, String fileName)
+    {
+        return embulk.loadYamlResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    private static String readResource(String fileName)
+    {
+        return EmbulkTests.readResource(BASIC_RESOURCE_PATH + fileName);
+    }
+
+    @Rule
+    public TestingEmbulk embulk = TestingEmbulk.builder()
+            .registerPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class)
+            .build();
+
+    private ConfigSource baseConfig;
+
+    @Before
+    public void setup()
+    {
+        baseConfig = MySQLTests.baseConfig();
+        execute(readResource("setup.sql")); // setup rows
+    }
+
+    @Test
+    public void testInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testInsertDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_insert_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testTruncateInsertBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_truncate_insert_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_truncate_insert_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    @Test
+    public void testMergeDirectBeforeLoad() throws Exception
+    {
+        execute("insert into test1 values('A002', 1, 'y')");
+        execute("insert into test1 values('A003', 1, 'y')");
+        execute("insert into test1 values('B001', 0, 'z')");
+        execute("insert into test1 values('B002', 9, 'z')");
+
+        Path in1 = toPath("test1.csv");
+        TestingEmbulk.RunResult result1 = embulk.runOutput(baseConfig.merge(loadYamlResource(embulk, "test_merge_direct_before_load.yml")), in1);
+        assertThat(selectRecords(embulk, "test1"), is(readResource("test_merge_before_load_expected.csv")));
+        //assertThat(result1.getConfigDiff(), is((ConfigDiff) loadYamlResource(embulk, "test_expected.diff")));
+    }
+
+    private Path toPath(String fileName) throws URISyntaxException
+    {
+        URL url = Resources.getResource(BASIC_RESOURCE_PATH + fileName);
+        return FileSystems.getDefault().getPath(new File(url.toURI()).getAbsolutePath());
+    }
+
+}

data/src/test/java/org/embulk/output/mysql/MySQLTests.java

@@ -0,0 +1,69 @@
+package org.embulk.output.mysql;
+
+import static java.util.Locale.ENGLISH;
+import static org.embulk.test.EmbulkTests.readSortedFile;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+
+import org.embulk.config.ConfigSource;
+import org.embulk.test.EmbulkTests;
+import org.embulk.test.TestingEmbulk;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.io.ByteStreams;
+
+public class MySQLTests
+{
+    public static ConfigSource baseConfig()
+    {
+        return EmbulkTests.config("EMBULK_OUTPUT_MYSQL_TEST_CONFIG");
+    }
+
+    public static void execute(String sql)
+    {
+        ConfigSource config = baseConfig();
+
+        ImmutableList.Builder<String> args = ImmutableList.builder();
+        args.add("mysql")
+                .add("-u")
+                .add(config.get(String.class, "user"));
+        if (!config.get(String.class, "password").isEmpty()) {
+            args.add("-p" + config.get(String.class, "password"));
+        }
+        args
+                .add("-h")
+                .add(config.get(String.class, "host"))
+                .add("-P")
+                .add(config.get(String.class, "port", "3306"))
+                .add(config.get(String.class, "database"))
+                .add("-e")
+                .add(sql);
+
+        ProcessBuilder pb = new ProcessBuilder(args.build());
+        pb.redirectErrorStream(true);
+        int code;
+        try {
+            Process process = pb.start();
+            ByteStreams.copy(process.getInputStream(), System.out);
+            code = process.waitFor();
+        } catch (IOException | InterruptedException ex) {
+            throw Throwables.propagate(ex);
+        }
+        if (code != 0) {
+            throw new RuntimeException(String.format(ENGLISH,
+                    "Command finished with non-zero exit code. Exit code is %d.", code));
+        }
+    }
+
+    public static String selectRecords(TestingEmbulk embulk, String tableName) throws IOException
+    {
+        Path temp = embulk.createTempFile("txt");
+        Files.delete(temp);
+        // test user needs FILE privilege
+        execute("select * from " + tableName + " into outfile '" + temp.toString().replace("\\", "\\\\") + "' fields terminated by ','");
+        return readSortedFile(temp);
+    }
+}

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/setup.sql

@@ -0,0 +1,8 @@
+drop table if exists test1;
+
+create table test1 (
+    id              char(4),
+    int_item        int,
+    varchar_item    varchar(8),
+    primary key (id)
+);

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test1.csv

@@ -0,0 +1,4 @@
+id:string,int_item:long,varchar_item:string
+A001,9,a
+A002,0,b
+A003,9,c

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_expected.diff

@@ -0,0 +1,2 @@
+in: {}
+out: {}

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: insert
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_after_load_expected.csv

@@ -0,0 +1,5 @@
+A001,9,x
+A002,0,b
+A003,9,x
+B001,0,z
+B002,9,x

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_insert_direct_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: insert_direct
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: merge
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_after_load_expected.csv

@@ -0,0 +1,5 @@
+A001,9,x
+A002,0,b
+A003,9,x
+B001,0,z
+B002,9,x

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_merge_direct_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: merge_direct
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_replace_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: replace
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_replace_after_load_expected.csv

@@ -0,0 +1,3 @@
+A001,9,x
+A002,0,b
+A003,9,x

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_truncate_insert_after_load.yml

@@ -0,0 +1,3 @@
+table: test1
+mode: truncate_insert
+after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/org/embulk/output/mysql/test/expect/after_load/test_truncate_insert_after_load_expected.csv

@@ -0,0 +1,3 @@
+A001,9,x
+A002,0,b
+A003,9,x