embulk-output-oracle 0.7.5 → 0.7.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 44663c92d50300afd8216bc26987e9723688eaef
-  data.tar.gz: 2d3379ffeb559914ab552dfd55bbd5a681feb8aa
+  metadata.gz: 6bae783332773b7e53b34d8676ad6592d4354618
+  data.tar.gz: cb529aeca71d8c03ba1009c491e8816d41814e6b
 SHA512:
-  metadata.gz: 834f776e6b4c08e781ce108f58402d0e83f1cfac21fbdc7bcd295e6f73e469f1628bec55f7fcc7e44dae154ea4a178501ebb4f5bc054f85a5140faefc775d945
-  data.tar.gz: f1102c5ebef6d1649543a181d1d47f4b0e56ac5186aea5a07433b0abc8472653104154c715c1a029de61da894793ba8f870f9e81c713c10ddea577632904d698
+  metadata.gz: c91850c74c82a2e5d52d4b0ba78f7ba6dd53f1b9a88efc933c0aeb22ae60091887410dade93de799d8fe6136e9c1e2289eea506d0bf2a67f202d0a9dc05c768c
+  data.tar.gz: 4a205e6af57486fb767e3c50825737008985dc184978cb0dbcb16c3da79ae11615329697265674a9529db98df8bb7b965cdf6033cb995b90f347a716512083f7

data/README.md

@@ -22,7 +22,9 @@ Oracle output plugin for Embulk loads records to Oracle.
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
-- **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See below. (string, required)
+- **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)
+- **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
+- **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = T.foo + S.foo` (`T` means target table and `S` means source table). (string array, default: always overwrites with new values)
 - **insert_method**: see below
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
@@ -51,6 +53,10 @@ Oracle output plugin for Embulk loads records to Oracle.
   * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
   * Transactional: Yes.
   * Resumable: No.
+* **merge**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `MERGE INTO ... WHEN MATCHED THEN UPDATE ...  WHEN NOT MATCHED THEN INSERT ...` query. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes.
+  * Resumable: Yes.
 
 ### Insert methods
 

data/src/main/java/org/embulk/output/OracleOutputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.output;
 
-import java.util.List;
 import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
@@ -70,7 +69,7 @@ public class OracleOutputPlugin
     {
         return new Features()
             .setMaxTableNameLength(30)
-            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
             .setIgnoreMergeKeys(false);
     }
 

data/src/main/java/org/embulk/output/oracle/OracleOutputConnection.java

@@ -8,11 +8,13 @@ import java.sql.SQLException;
 import java.sql.Statement;
 import java.util.Arrays;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.MergeConfig;
 
 public class OracleOutputConnection
         extends JdbcOutputConnection
@@ -195,4 +197,73 @@ public class OracleOutputConnection
         }
         return super.getColumnDeclareType(convertedTypeName, col);
     }
+
+    @Override
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("MERGE INTO ");
+        sb.append(quoteIdentifierString(toTable));
+        sb.append(" T");
+        sb.append(" USING (");
+        for (int i = 0; i < fromTables.size(); i++) {
+            if (i != 0) { sb.append(" UNION ALL "); }
+            sb.append(" SELECT ");
+            sb.append(buildColumns(schema, ""));
+            sb.append(" FROM ");
+            sb.append(quoteIdentifierString(fromTables.get(i)));
+        }
+        sb.append(") S");
+        sb.append(" ON (");
+        for (int i = 0; i < mergeConfig.getMergeKeys().size(); i++) {
+            if (i != 0) { sb.append(" AND "); }
+            String mergeKey = quoteIdentifierString(mergeConfig.getMergeKeys().get(i));
+            sb.append("T.");
+            sb.append(mergeKey);
+            sb.append(" = S.");
+            sb.append(mergeKey);
+        }
+        sb.append(")");
+        sb.append(" WHEN MATCHED THEN");
+        sb.append(" UPDATE SET ");
+        if (mergeConfig.getMergeRule().isPresent()) {
+            for (int i = 0; i < mergeConfig.getMergeRule().get().size(); i++) {
+                if (i != 0) { sb.append(", "); }
+                sb.append(mergeConfig.getMergeRule().get().get(i));
+            }
+        } else {
+            int index = 0;
+            for (int i = 0; i < schema.getCount(); i++) {
+                String rawColumn = schema.getColumnName(i);
+                if (mergeConfig.getMergeKeys().contains(rawColumn)) {
+                    continue;
+                }
+                if (index++ != 0) { sb.append(", "); }
+                String column = quoteIdentifierString(rawColumn);
+                sb.append(column);
+                sb.append(" = S.");
+                sb.append(column);
+            }
+        }
+        sb.append(" WHEN NOT MATCHED THEN");
+        sb.append(" INSERT (");
+        sb.append(buildColumns(schema, ""));
+        sb.append(") VALUES (");
+        sb.append(buildColumns(schema, "S."));
+        sb.append(")");
+
+        return sb.toString();
+    }
+
+    private String buildColumns(JdbcSchema schema, String prefix)
+    {
+        StringBuilder sb = new StringBuilder();
+        for (int i = 0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            sb.append(prefix);
+            sb.append(quoteIdentifierString(schema.getColumnName(i)));
+        }
+        return sb.toString();
+    }
 }

data/src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java

@@ -329,6 +329,195 @@ public class OracleOutputPluginTest extends AbstractJdbcOutputPluginTest
         assertTable(table);
     }
 
+    @Test
+    public void testMerge() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST1";
+
+        dropTable(table);
+        executeSQL("CREATE TABLE TEST1 ("
+                + "ID              CHAR(4),"
+                + "VARCHAR2_ITEM   VARCHAR2(6),"
+                + "NUMBER_ITEM     NUMBER(10,2),"
+                + "PRIMARY KEY (ID))");
+        executeSQL("INSERT INTO TEST1 VALUES('A001', 'AAA', 12.34)");
+        executeSQL("INSERT INTO TEST1 VALUES('A003', NULL, NULL)");
+        executeSQL("INSERT INTO TEST1 VALUES('A005', 'EEE', 56.78)");
+        executeSQL("INSERT INTO TEST1 VALUES('A006', 'FFF', 0)");
+
+        test("/oracle/yml/test-merge.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(6, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals("aaa", i2.next());
+            assertEquals(new BigDecimal("99.99"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals("bbb", i2.next());
+            assertEquals(new BigDecimal("88.88"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals("ccc", i2.next());
+            assertEquals(new BigDecimal("77.77"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A004", i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A005", i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A006", i2.next());
+            assertEquals("FFF", i2.next());
+            assertEquals(new BigDecimal("0"), i2.next());
+        }
+    }
+
+    @Test
+    public void testMergeWithKeys() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST1";
+
+        dropTable(table);
+        executeSQL("CREATE TABLE TEST1 ("
+                + "ID              CHAR(4),"
+                + "VARCHAR2_ITEM   VARCHAR2(6),"
+                + "NUMBER_ITEM     NUMBER(10,2),"
+                + "PRIMARY KEY (ID))");
+        executeSQL("INSERT INTO TEST1 VALUES('A001', 'AAA', 11.11)");
+        executeSQL("INSERT INTO TEST1 VALUES('A002', 'BBB', 22.22)");
+        executeSQL("INSERT INTO TEST1 VALUES('A003', 'CCC', 33.33)");
+
+        test("/oracle/yml/test-merge-keys.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(5, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals("AAA", i2.next());
+            assertEquals(new BigDecimal("11.11"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A004", i2.next());
+            assertEquals("BBB", i2.next());
+            assertEquals(new BigDecimal("22.21"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A005", i2.next());
+            assertEquals("BBB", i2.next());
+            assertEquals(new BigDecimal("22.22"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A006", i2.next());
+            assertEquals("BBB", i2.next());
+            assertEquals(new BigDecimal("22.23"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A007", i2.next());
+            assertEquals("CCC", i2.next());
+            assertEquals(new BigDecimal("33.33"), i2.next());
+        }
+    }
+
+    @Test
+    public void testMergeWithRule() throws Exception
+    {
+        if (!enabled) {
+            return;
+        }
+
+        String table = "TEST1";
+
+        dropTable(table);
+        executeSQL("CREATE TABLE TEST1 ("
+                + "ID              CHAR(4),"
+                + "VARCHAR2_ITEM   VARCHAR2(6),"
+                + "NUMBER_ITEM     NUMBER(10,2),"
+                + "PRIMARY KEY (ID))");
+        executeSQL("INSERT INTO TEST1 VALUES('A002', 'BBB', 22.22)");
+        executeSQL("INSERT INTO TEST1 VALUES('A004', 'DDD', 44.44)");
+        executeSQL("INSERT INTO TEST1 VALUES('A006', 'FFF', 66.66)");
+
+        /*
+        A001,aaa,99.99
+        A002,bbb,88.88
+        A003,ccc,77.77
+        A004,,
+        A005,,
+*/
+
+        test("/oracle/yml/test-merge-rule.yml");
+
+        List<List<Object>> rows = select(table);
+        assertEquals(6, rows.size());
+        Iterator<List<Object>> i1 = rows.iterator();
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A001", i2.next());
+            assertEquals("aaa", i2.next());
+            assertEquals(new BigDecimal("99.99"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A002", i2.next());
+            assertEquals("x", i2.next());
+            assertEquals(new BigDecimal("111.1"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A003", i2.next());
+            assertEquals("ccc", i2.next());
+            assertEquals(new BigDecimal("77.77"), i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A004", i2.next());
+            assertEquals("x", i2.next());
+            assertEquals(null, i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A005", i2.next());
+            assertEquals(null, i2.next());
+            assertEquals(null, i2.next());
+        }
+        {
+            Iterator<Object> i2 = i1.next().iterator();
+            assertEquals("A006", i2.next());
+            assertEquals("FFF", i2.next());
+            assertEquals(new BigDecimal("66.66"), i2.next());
+        }
+    }
+
     @Test
     public void testUrl() throws Exception
     {

data/src/test/resources/oracle/data/test4/test4.csv

@@ -0,0 +1,5 @@
+A001,aaa,99.99
+A002,bbb,88.88
+A003,ccc,77.77
+A004,,
+A005,,

data/src/test/resources/oracle/data/test5/test5.csv

@@ -0,0 +1,4 @@
+A004,BBB,22.21
+A005,BBB,22.22
+A006,BBB,22.23
+A007,CCC,33.33

data/src/test/resources/oracle/yml/test-merge-keys.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: '/oracle/data/test5/test5.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: VARCHAR2_ITEM, type: string}
+    - {name: NUMBER_ITEM, type: string}
+out:
+    type: oracle
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST1
+    mode: merge
+    merge_keys: [VARCHAR2_ITEM,NUMBER_ITEM]
+    #driver_path: driver/ojdbc7.jar

data/src/test/resources/oracle/yml/test-merge-rule.yml

@@ -0,0 +1,23 @@
+in:
+  type: file
+  path_prefix: '/oracle/data/test4/test4.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: VARCHAR2_ITEM, type: string}
+    - {name: NUMBER_ITEM, type: string}
+out:
+    type: oracle
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST1
+    mode: merge
+    merge_rule: ["VARCHAR2_ITEM = 'x'", "NUMBER_ITEM = T.NUMBER_ITEM + S.NUMBER_ITEM"]
+    #driver_path: driver/ojdbc7.jar

data/src/test/resources/oracle/yml/test-merge.yml

@@ -0,0 +1,22 @@
+in:
+  type: file
+  path_prefix: '/oracle/data/test4/test4.csv'
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: ''
+    columns:
+    - {name: ID, type: string}
+    - {name: VARCHAR2_ITEM, type: string}
+    - {name: NUMBER_ITEM, type: string}
+out:
+    type: oracle
+    host: #host#
+    database: #database#
+    user: #user#
+    password: #password#
+    table: TEST1
+    mode: merge
+    #driver_path: driver/ojdbc7.jar

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-oracle
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-10 00:00:00.000000000 Z
+date: 2017-03-23 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.5.jar
-- classpath/embulk-output-oracle-0.7.5.jar
+- classpath/embulk-output-jdbc-0.7.6.jar
+- classpath/embulk-output-oracle-0.7.6.jar
 - lib/embulk/native/x86_64-linux/libembulk-output-oracle-oci.so
 - lib/embulk/native/x86_64-windows/embulk-output-oracle-oci.dll
 - lib/embulk/output/oracle.rb
@@ -50,6 +50,8 @@ files:
 - src/test/java/org/embulk/output/oracle/OracleOutputPluginTest.java
 - src/test/resources/oracle/data/test1/test1.csv
 - src/test/resources/oracle/data/test3/test3.csv
+- src/test/resources/oracle/data/test4/test4.csv
+- src/test/resources/oracle/data/test5/test5.csv
 - src/test/resources/oracle/yml/test-insert-direct-direct-method.yml
 - src/test/resources/oracle/yml/test-insert-direct-empty.yml
 - src/test/resources/oracle/yml/test-insert-direct-oci-method-large.yml
@@ -62,6 +64,9 @@ files:
 - src/test/resources/oracle/yml/test-lower-column-options.yml
 - src/test/resources/oracle/yml/test-lower-column.yml
 - src/test/resources/oracle/yml/test-lower-table.yml
+- src/test/resources/oracle/yml/test-merge-keys.yml
+- src/test/resources/oracle/yml/test-merge-rule.yml
+- src/test/resources/oracle/yml/test-merge.yml
 - src/test/resources/oracle/yml/test-replace-empty.yml
 - src/test/resources/oracle/yml/test-replace-long-name-multibyte.yml
 - src/test/resources/oracle/yml/test-replace-long-name.yml