embulk-output-sqlserver 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d974a42bda50d1cc40c33c3d25e15ef4a74fbc91
-  data.tar.gz: c9f3cc3c294b6528344aed949cd991cd5d781f48
+  metadata.gz: 83928aeb58e8e44cd1e3ddd359a5f0bc91a8b480
+  data.tar.gz: ebcde6ea3c960f1a962750e41e43732fe9bcff60
 SHA512:
-  metadata.gz: a6460579ddcf2544670f3fef6a8a8e376cfc39bb2c5aef1efb167c5863832f2f8792dd81769cf767814135292cf9ca79496a173174252246686bb3635608c8ef
-  data.tar.gz: 56502e2371187fdeeb45d0f15ce5687d549a86cba5122fda35273fef6f6fe6eecba48744a2f2c2f6d583cad317a845a8363d7f8996868d386b499de1d82b5eac
+  metadata.gz: 0321e417ca209e078b1c9f183f3332f23068f7b6ae0abfd2115f2c15ebbd3c6aa9b6a0b97c2f6e45a44114fd462234e6765e3c882929fa76d91fd2cc651141e6
+  data.tar.gz: 8a939954de682d79658f854fd62ccbd42857be89c579c7ed22b1d44993c2ed0653910d6e4529bd82403558ca05b92c7be1999257d369908f665b965fe0fb6937

data/README.md

@@ -28,7 +28,9 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
-- **mode**: "insert", "insert_direct", "truncate_insert" or "replace". See below. (string, required)
+- **mode**: "insert", "insert_direct", "truncate_insert" , "replace" or "merge". See below. (string, required)
+- **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
+- **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = T.foo + S.foo` (`T` means target table and `S` means source table). (string array, default: always overwrites with new values)
 - **insert_method**: see below
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
@@ -57,6 +59,10 @@ embulk "-J-Djava.library.path=C:\drivers" run input-sqlserver.yml
   * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
   * Transactional: No. If fails, the target table could be dropped (because SQL Server can't rollback DDL).
   * Resumable: No.
+* **merge**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `MERGE INTO ... WHEN MATCHED THEN UPDATE ...  WHEN NOT MATCHED THEN INSERT ...` query. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes.
+  * Resumable: Yes.
 
 ### Insert methods
 

data/src/main/java/org/embulk/output/SQLServerOutputPlugin.java

@@ -2,6 +2,7 @@ package org.embulk.output;
 
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
+
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigException;
@@ -103,7 +104,7 @@ public class SQLServerOutputPlugin
     {
         return new Features()
             .setMaxTableNameLength(128)
-            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
             .setIgnoreMergeKeys(false);
     }
 

data/src/main/java/org/embulk/output/sqlserver/SQLServerOutputConnection.java

@@ -4,10 +4,12 @@ import java.sql.Connection;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
+import java.util.List;
 
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.MergeConfig;
 
 public class SQLServerOutputConnection
         extends JdbcOutputConnection
@@ -113,12 +115,67 @@ public class SQLServerOutputConnection
         return super.getColumnDeclareType(convertedTypeName, col);
     }
 
-    /*
     @Override
-    public Charset getTableNameCharset() throws SQLException
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {
-        return getOracleCharset().getJavaCharset();
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("MERGE INTO ");
+        sb.append(quoteIdentifierString(toTable));
+        sb.append(" AS T");
+        sb.append(" USING (");
+        for (int i = 0; i < fromTables.size(); i++) {
+            if (i != 0) { sb.append(" UNION ALL "); }
+            sb.append(" SELECT ");
+            sb.append(buildColumns(schema, ""));
+            sb.append(" FROM ");
+            sb.append(quoteIdentifierString(fromTables.get(i)));
+        }
+        sb.append(") AS S");
+        sb.append(" ON (");
+        for (int i = 0; i < mergeConfig.getMergeKeys().size(); i++) {
+            if (i != 0) { sb.append(" AND "); }
+            String mergeKey = quoteIdentifierString(mergeConfig.getMergeKeys().get(i));
+            sb.append("T.");
+            sb.append(mergeKey);
+            sb.append(" = S.");
+            sb.append(mergeKey);
+        }
+        sb.append(")");
+        sb.append(" WHEN MATCHED THEN");
+        sb.append(" UPDATE SET ");
+        if (mergeConfig.getMergeRule().isPresent()) {
+            for (int i = 0; i < mergeConfig.getMergeRule().get().size(); i++) {
+                if (i != 0) { sb.append(", "); }
+                sb.append(mergeConfig.getMergeRule().get().get(i));
+            }
+        } else {
+            for (int i = 0; i < schema.getCount(); i++) {
+                if (i != 0) { sb.append(", "); }
+                String column = quoteIdentifierString(schema.getColumnName(i));
+                sb.append(column);
+                sb.append(" = S.");
+                sb.append(column);
+            }
+        }
+        sb.append(" WHEN NOT MATCHED THEN");
+        sb.append(" INSERT (");
+        sb.append(buildColumns(schema, ""));
+        sb.append(") VALUES (");
+        sb.append(buildColumns(schema, "S."));
+        sb.append(");");
+
+        return sb.toString();
     }
 
-    */
+    private String buildColumns(JdbcSchema schema, String prefix)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            sb.append(prefix);
+            sb.append(quoteIdentifierString(schema.getColumnName(i)));
+        }
+        return sb.toString();
+    }
 }

data/src/test/java/org/embulk/output/sqlserver/SQLServerOutputPluginTest.java

@@ -34,7 +34,7 @@ public class SQLServerOutputPluginTest extends AbstractJdbcOutputPluginTest
         try {
             Class.forName("com.microsoft.sqlserver.jdbc.SQLServerDriver");
         } catch (ClassNotFoundException e) {
-            System.err.println("Warning: you should put 'sqljdbc41.jar' in 'embulk-input-sqlserver/driver' directory in order to test.");
+            System.err.println("Warning: you should put 'sqljdbc41.jar' in 'embulk-output-sqlserver/driver' directory in order to test.");
             return;
         }
 
@@ -358,6 +358,145 @@ public class SQLServerOutputPluginTest extends AbstractJdbcOutputPluginTest
         }
     }
 
+    @Test
+    public void testMerge() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST6";
+
+        dropTable(table);
+        executeSQL(String.format("CREATE TABLE %S (ITEM1 INT, ITEM2 INT, ITEM3 VARCHAR(4), PRIMARY KEY(ITEM1, ITEM2))", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 20, 'A')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 21, 'B')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(11, 20, 'C')", table));
+
+        test("/sqlserver/yml/test-merge.yml");
+
+        assertMergedTable(table);
+    }
+
+    @Test
+    public void testMergeWithKeys() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST6b";
+
+        dropTable(table);
+        executeSQL(String.format("CREATE TABLE %S (ITEM1 INT, ITEM2 INT, ITEM3 VARCHAR(4))", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 20, 'A')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 21, 'B')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(11, 20, 'C')", table));
+
+        test("/sqlserver/yml/test-merge-keys.yml");
+
+        assertMergedTable(table);
+    }
+
+    @Test
+    public void testMergeWithRule() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST6";
+
+        dropTable(table);
+        executeSQL(String.format("CREATE TABLE %S (ITEM1 INT, ITEM2 INT, ITEM3 VARCHAR(4), PRIMARY KEY(ITEM1, ITEM2))", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 20, 'A')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(10, 21, 'B')", table));
+        executeSQL(String.format("INSERT INTO %S VALUES(11, 20, 'C')", table));
+
+        test("/sqlserver/yml/test-merge-rule.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(6, rows.size());
+        {
+            List<Object> row = rows.get(0);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("A", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(1);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(21), row.get(1));
+            assertEquals("Baa", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(2);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(22), row.get(1));
+            assertEquals("dd", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(3);
+            assertEquals(Integer.valueOf(11), row.get(0));
+            assertEquals(Integer.valueOf(10), row.get(1));
+            assertEquals("bb", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(4);
+            assertEquals(Integer.valueOf(11), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("C", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(5);
+            assertEquals(Integer.valueOf(12), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("cc", row.get(2));
+        }
+    }
+
+    private void assertMergedTable(String table) throws Exception
+    {
+        List<List<Object>> rows = select(table);
+        assertEquals(6, rows.size());
+        {
+            List<Object> row = rows.get(0);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("A", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(1);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(21), row.get(1));
+            assertEquals("aa", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(2);
+            assertEquals(Integer.valueOf(10), row.get(0));
+            assertEquals(Integer.valueOf(22), row.get(1));
+            assertEquals("dd", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(3);
+            assertEquals(Integer.valueOf(11), row.get(0));
+            assertEquals(Integer.valueOf(10), row.get(1));
+            assertEquals("bb", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(4);
+            assertEquals(Integer.valueOf(11), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("C", row.get(2));
+        }
+        {
+            List<Object> row = rows.get(5);
+            assertEquals(Integer.valueOf(12), row.get(0));
+            assertEquals(Integer.valueOf(20), row.get(1));
+            assertEquals("cc", row.get(2));
+        }
+    }
+
 
     private void assertTable(int skip, String table) throws Exception
     {

data/src/test/resources/sqlserver/data/test6/test6.csv

@@ -0,0 +1,4 @@
+10,21,aa
+11,10,bb
+12,20,cc
+10,22,dd

data/src/test/resources/sqlserver/yml/test-merge-keys.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: '/sqlserver/data/test6/test6.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ITEM1, type: long}
+    - {name: ITEM2, type: long}
+    - {name: ITEM3, type: string}
+out:
+    type: sqlserver
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST6B
+    mode: merge
+    merge_keys: [ITEM1,ITEM2]
+

data/src/test/resources/sqlserver/yml/test-merge-rule.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/sqlserver/data/test6/test6.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ITEM1, type: long}
+    - {name: ITEM2, type: long}
+    - {name: ITEM3, type: string}
+out:
+    type: sqlserver
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST6
+    mode: merge
+    merge_rule: ["ITEM3 = T.ITEM3 + S.ITEM3"]

data/src/test/resources/sqlserver/yml/test-merge.yml

@@ -0,0 +1,21 @@
+in:
+  type: file
+  path_prefix: '/sqlserver/data/test6/test6.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ITEM1, type: long}
+    - {name: ITEM2, type: long}
+    - {name: ITEM3, type: string}
+out:
+    type: sqlserver
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST6
+    mode: merge

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sqlserver
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-25 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.1.jar
-- classpath/embulk-output-sqlserver-0.7.1.jar
+- classpath/embulk-output-jdbc-0.7.2.jar
+- classpath/embulk-output-sqlserver-0.7.2.jar
 - classpath/jtds-1.3.1.jar
 - lib/embulk/output/sqlserver.rb
 - src/main/java/org/embulk/output/SQLServerOutputPlugin.java
@@ -42,9 +42,13 @@ files:
 - src/test/resources/sqlserver/data/test3/test3.csv
 - src/test/resources/sqlserver/data/test4/test4.csv
 - src/test/resources/sqlserver/data/test5/test5.csv
+- src/test/resources/sqlserver/data/test6/test6.csv
 - src/test/resources/sqlserver/yml/test-insert-direct.yml
 - src/test/resources/sqlserver/yml/test-insert.yml
 - src/test/resources/sqlserver/yml/test-jtds.yml
+- src/test/resources/sqlserver/yml/test-merge-keys.yml
+- src/test/resources/sqlserver/yml/test-merge-rule.yml
+- src/test/resources/sqlserver/yml/test-merge.yml
 - src/test/resources/sqlserver/yml/test-native-date.yml
 - src/test/resources/sqlserver/yml/test-native-decimal.yml
 - src/test/resources/sqlserver/yml/test-native-integer.yml