embulk-output-mysql 0.7.0 → 0.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bedcc590df0bc10158116c37710c92cf6f737ce6
-  data.tar.gz: 6ce3d054a7a9e875238fb554b389f2dc29ef790b
+  metadata.gz: 9830d9379f4a4e6e2db01c7048a651b8db696fe2
+  data.tar.gz: ce2688a78d31c9e8b417da61ed4e7e4019b289ec
 SHA512:
-  metadata.gz: 11418af4cf4d768877c63ae51f2aa66aff48895d3e7ffb51a94d2359375b96deb1e8e78055b7f099a2dffbdc00dced37c43ada67219620289255dc7cc7ed5d76
-  data.tar.gz: 8653fa077f225e7f32ace989c48dec9a6b36b07249fb93f8982688752a236a0e72d7b31dab3b76eaaf10441c262be4df8061c8271d7ce4e06a258e08011dc1e6
+  metadata.gz: 704b948e549ee8a5376558b99688da68742e1c098bb983eb5c6a30e73f4aa5050d15cac526156b1f4172108f185456b6b2bae73cac6817c12aafb5b1cd9b924a
+  data.tar.gz: 8329a028727fc4dab736663291c83ae511396cb4c85b282983d1584ffa94048b7f44fa05a51e0b113f2fcdfee489453cbc98e9892177962d008920a7b05061c5

data/README.md

@@ -29,6 +29,7 @@ MySQL output plugin for Embulk loads records to MySQL.
   - **value_type**: This plugin converts input column type (embulk type) into a database type to build a INSERT statement. This value_type option controls the type of the value in a INSERT statement. (string, default: depends on the sql type of the column. Available values options are: `byte`, `short`, `int`, `long`, `double`, `float`, `boolean`, `string`, `nstring`, `date`, `time`, `timestamp`, `decimal`, `json`, `null`, `pass`)
   - **timestamp_format**: If input column type (embulk type) is timestamp and value_type is `string` or `nstring`, this plugin needs to format the timestamp value into a string. This timestamp_format option is used to control the format of the timestamp. (string, default: `%Y-%m-%d %H:%M:%S.%6N`)
   - **timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp value into a SQL string. In this cases, this timezone option is used to control the timezone. (string, value of default_timezone option is used by default)
+- **after_load**: if set, this SQL will be executed after loading all records.
 
 ### Modes
 

data/build.gradle

@@ -1,7 +1,7 @@
 dependencies {
     compile project(':embulk-output-jdbc')
-
     compile 'mysql:mysql-connector-java:5.1.34'
 
+    testCompile 'org.embulk:embulk-standards:0.8.8'
     testCompile project(':embulk-output-jdbc').sourceSets.test.output
 }

data/src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java

@@ -0,0 +1,340 @@
+package org.embulk.output.mysql;
+
+import static java.util.Locale.ENGLISH;
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.Iterator;
+import java.util.List;
+
+import org.embulk.output.AbstractJdbcOutputPluginTest;
+import org.embulk.output.MySQLOutputPlugin;
+import org.embulk.spi.OutputPlugin;
+import org.junit.Test;
+
+
+public class MySQLOutputPluginTest extends AbstractJdbcOutputPluginTest
+{
+    @Override
+    protected void prepare() throws SQLException
+    {
+        tester.addPlugin(OutputPlugin.class, "mysql", MySQLOutputPlugin.class);
+
+        try {
+            connect();
+        } catch (SQLException e) {
+            System.err.println(e);
+            System.err.println(String.format(ENGLISH, "Warning: prepare a schema on MySQL (server = %s, port = %d, database = %s, user = %s, password = %s).",
+                    getHost(), getPort(), getDatabase(), getUser(), getPassword()));
+            return;
+        }
+
+        enabled = true;
+    }
+
+    @Test
+    public void testInsertDirectAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-insert-direct-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(5, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B001", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("z", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B002", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    @Test
+    public void testInsertAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-insert-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(5, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B001", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("z", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B002", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    @Test
+    public void testTruncateInsertAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-truncate-insert-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(3, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    @Test
+    public void testReplaceAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-replace-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(3, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9L, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0L, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9L, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    @Test
+    public void testMergeDirectAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-merge-direct-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(5, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B001", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("z", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B002", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    @Test
+    public void testMergeAfterLoad() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "test1";
+
+        dropTable(table);
+        createTable(table);
+        executeSQL(String.format("INSERT INTO %s VALUES('A002', 1, 'y')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('A003', 1, 'y')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B001', 0, 'z')", table));
+        executeSQL(String.format("INSERT INTO %s VALUES('B002', 9, 'z')", table));
+
+        test("/mysql/yml/test-merge-after-load.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(5, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("b", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B001", i2.next());
+            assertEquals(0, i2.next());
+            assertEquals("z", i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("B002", i2.next());
+            assertEquals(9, i2.next());
+            assertEquals("x", i2.next());
+        }
+    }
+
+    private void createTable(String table) throws SQLException
+    {
+        String sql = String.format("create table %s ("
+                + "id              char(4),"
+                + "int_item        int,"
+                + "varchar_item    varchar(8),"
+                + "primary key (id))", table);
+        executeSQL(sql);
+    }
+
+    @Override
+    protected Connection connect() throws SQLException
+    {
+        return DriverManager.getConnection(String.format(ENGLISH, "jdbc:mysql://%s:%d/%s", getHost(), getPort(), getDatabase()),
+                getUser(), getPassword());
+    }
+
+}

data/src/test/resources/mysql/data/test1.csv

@@ -0,0 +1,3 @@
+A001,9,a
+A002,0,b
+A003,9,c

data/src/test/resources/mysql/yml/test-insert-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: insert
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/mysql/yml/test-insert-direct-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: insert_direct
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/mysql/yml/test-merge-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: merge
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/mysql/yml/test-merge-direct-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: merge_direct
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/mysql/yml/test-replace-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: replace
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

data/src/test/resources/mysql/yml/test-truncate-insert-after-load.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/mysql/data/test1.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: id, type: string}
+    - {name: int_item, type: long}
+    - {name: varchar_item, type: string}
+out:
+    type: mysql
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: test1
+    mode: truncate_insert
+    after_load: "update test1 set varchar_item = 'x' where int_item = 9"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-mysql
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-26 00:00:00.000000000 Z
+date: 2016-11-25 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,14 +19,22 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.0.jar
-- classpath/embulk-output-mysql-0.7.0.jar
+- classpath/embulk-output-jdbc-0.7.1.jar
+- classpath/embulk-output-mysql-0.7.1.jar
 - classpath/mysql-connector-java-5.1.34.jar
 - lib/embulk/output/mysql.rb
 - src/main/java/org/embulk/output/MySQLOutputPlugin.java
 - src/main/java/org/embulk/output/mysql/MySQLBatchInsert.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnection.java
 - src/main/java/org/embulk/output/mysql/MySQLOutputConnector.java
+- src/test/java/org/embulk/output/mysql/MySQLOutputPluginTest.java
+- src/test/resources/mysql/data/test1.csv
+- src/test/resources/mysql/yml/test-insert-after-load.yml
+- src/test/resources/mysql/yml/test-insert-direct-after-load.yml
+- src/test/resources/mysql/yml/test-merge-after-load.yml
+- src/test/resources/mysql/yml/test-merge-direct-after-load.yml
+- src/test/resources/mysql/yml/test-replace-after-load.yml
+- src/test/resources/mysql/yml/test-truncate-insert-after-load.yml
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0