embulk-output-postgresql 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b89423ada5663fef11f6ee4004a3d71521ee5733
-  data.tar.gz: 23c8d90ed1a630ab7859652682be4a6ec27cb178
+  metadata.gz: 7acb9bf51cfa9656c136a38727e5730c142e8d14
+  data.tar.gz: 31cc497361027727ff1526049962e6772b65bb13
 SHA512:
-  metadata.gz: 4d42ebf848fb2c8375802a2fd5d66afbb4e650d93c8f84cba94cc64de1dbf0d32e173474601c84719a37c69d240eaee9494d053b38fc1aa0dfd4fe22ced9946d
-  data.tar.gz: c47f7441ac94761138de35240fa25c70fef4dfefda3a43490f8c048186bce20ae7ffa4b443dc29c34f5122bb7bc403daf4cae603b5eae4d898dce23a98077257
+  metadata.gz: 80812f48c9dcb3530cb47a712b7c2b501d0859c6ada431ce4db4f109292baeb19f3dc274ada6e58bb7f58378d667997d95f47e887244d9c06cd22a377a78f82c
+  data.tar.gz: 6292455416b8487cdc84a4ad485ac351aeecf97656a7f1194dc15a0deda257f41cbda6f41160c95a618c213df44a32e5d47f2190eab8c079e9da0b4f470aee65

data/README.md

@@ -21,7 +21,7 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
-- **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)
+- **mode**: "insert", "insert_direct", "truncate_insert", "replace", "merge" or "merge_direct". See below. (string, required)
 - **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode if table doesn't have primary key)
 - **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = foo + S.foo` (`S` means source table). (string array, default: always overwrites with new values)
 - **ssl**: enables SSL. data will be encrypted but CA or certification will not be verified (boolean, default: false)
@@ -53,9 +53,13 @@ PostgreSQL output plugin for Embulk loads records to PostgreSQL.
   * Transactional: Yes.
   * Resumable: No.
 * **merge**:
-  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `with updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, runs `WITH updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
   * Transactional: Yes.
   * Resumable: Yes.
+* **merge_direct**:
+  * Behavior: This mode inserts rows to the target table directly using `WITH S AS (SELECT ...), updated AS (UPDATE .... RETURNING ...) INSERT INTO ....` query. If the target table doesn't exist, it is created automatically.
+  * Transactional: No.
+  * Resumable: No.
 
 ### Supported types
 

data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java

@@ -64,7 +64,7 @@ public class PostgreSQLOutputPlugin
     {
         return new Features()
             .setMaxTableNameLength(30)
-            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.MERGE, Mode.MERGE_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
             .setIgnoreMergeKeys(false);
     }
 
@@ -111,7 +111,7 @@ public class PostgreSQLOutputPlugin
     protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
         if (mergeConfig.isPresent()) {
-            throw new UnsupportedOperationException("PostgreSQL output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
+            return new StandardBatchInsert(getConnector(task, true), mergeConfig);
         }
         return new PostgreSQLCopyBatchInsert(getConnector(task, true));
     }

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java

@@ -44,6 +44,86 @@ public class PostgreSQLOutputConnection
         return new CopyManager((BaseConnection) connection);
     }
 
+    @Override
+    protected String buildPreparedMergeSql(String toTable, JdbcSchema schema, MergeConfig mergeConfig) throws SQLException
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("WITH S AS (");
+        sb.append("SELECT ");
+        for (int i = 0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            sb.append("? AS ");
+            quoteIdentifierString(sb, schema.getColumnName(i));
+        }
+        sb.append("),");
+        sb.append("updated AS (");
+        sb.append("UPDATE ");
+        quoteIdentifierString(sb, toTable);
+        sb.append(" SET ");
+        if (mergeConfig.getMergeRule().isPresent()) {
+            List<String> rule = mergeConfig.getMergeRule().get();
+            for (int i = 0; i < rule.size(); i++) {
+                if (i != 0) { sb.append(", "); }
+                sb.append(rule.get(i));
+            }
+        } else {
+            for (int i = 0; i < schema.getCount(); i++) {
+                if (i != 0) { sb.append(", "); }
+                quoteIdentifierString(sb, schema.getColumnName(i));
+                sb.append(" = S.");
+                quoteIdentifierString(sb, schema.getColumnName(i));
+            }
+        }
+        sb.append(" FROM S");
+        sb.append(" WHERE ");
+        List<String> mergeKeys = mergeConfig.getMergeKeys();
+        for (int i = 0; i < mergeKeys.size(); i++) {
+            if (i != 0) { sb.append(" AND "); }
+            quoteIdentifierString(sb, toTable);
+            sb.append(".");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+            sb.append(" = ");
+            sb.append("S.");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+        }
+        sb.append(" RETURNING ");
+        for (int i = 0; i < mergeKeys.size(); i++) {
+            if (i != 0) { sb.append(", "); }
+            sb.append("S.");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+        }
+        sb.append(") ");
+
+        sb.append("INSERT INTO ");
+        quoteIdentifierString(sb, toTable);
+        sb.append(" (");
+        for (int i = 0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, schema.getColumnName(i));
+        }
+        sb.append(") ");
+        sb.append("SELECT ");
+        for(int i = 0; i < schema.getCount(); i++) {
+            if (i != 0) { sb.append(", "); }
+            quoteIdentifierString(sb, schema.getColumnName(i));
+        }
+        sb.append(" FROM S ");
+        sb.append("WHERE NOT EXISTS (");
+        sb.append("SELECT 1 FROM updated WHERE ");
+        for (int i = 0; i < mergeKeys.size(); i++) {
+            if (i != 0) { sb.append(" AND "); }
+            sb.append("S.");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+            sb.append(" = ");
+            sb.append("updated.");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+        }
+        sb.append(") ");
+
+        return sb.toString();
+    }
+
     @Override
     protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-19 00:00:00.000000000 Z
+date: 2017-01-06 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,8 +19,8 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/embulk-output-jdbc-0.7.2.jar
-- classpath/embulk-output-postgresql-0.7.2.jar
+- classpath/embulk-output-jdbc-0.7.3.jar
+- classpath/embulk-output-postgresql-0.7.3.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/output/postgresql.rb
 - src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java