embulk-output-postgresql 0.6.1 → 0.6.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4779498654aaae70ab01a04cebca45eb702f0d55
-  data.tar.gz: 30c43767822c55f73e0d58443ba1a5e5615222e0
+  metadata.gz: 8613dcf9023886902ee147f03579177c56d7c456
+  data.tar.gz: 6c5967b4f4a805964b6a75e3eb79354b2e0068e3
 SHA512:
-  metadata.gz: ec2a3b15338f630ba5e047a8f0c1916ae9c6a0bf2e66b528ab715ca57e4fd874ca3347f9e60027671ef61496eb9d930478dc60ca64a931b9890354a370744aad
-  data.tar.gz: f52e2cb4b329f77f87c253b1a3874b2ebb5905180c29704457c8455c487d3886545e214f1326206f3d8500dba46f4c4c02e1b00128d09fb05e3dbccc97e0e222
+  metadata.gz: cb0c90dfe2353d952410c6eab6bb32d740c22e35f41fe13aced146af9cee5ab263b1a6a35c3a96fbcd163793072882a9ccd613b932b34dd16893bc9454e50167
+  data.tar.gz: dbdf6e39dc58639559e6035e7f935e74d401c8600ca739904e264037de034672420809d452147d6cad16faf733af41189ff6972656a5706eca88c1214eba2a8e

data/README.md

@@ -5,8 +5,8 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
 ## Overview
 
 * **Plugin type**: output
-* **Load all or nothing**: depnds on the mode. see below.
-* **Resume supported**: depnds on the mode. see below.
+* **Load all or nothing**: depends on the mode. see below.
+* **Resume supported**: depends on the mode. see below.
 
 ## Configuration
 
@@ -18,8 +18,12 @@ PostgreSQL output plugins for Embulk loads records to PostgreSQL.
 - **schema**: destination schema name (string, default: "public")
 - **table**: destination table name (string, required)
 - **options**: extra connection properties (hash, default: {})
+- **retry_limit** max retry count for database operations (integer, default: 12)
+- **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))
+- **max_retry_wait** upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)
 - **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode)
+- **merge_rule**: list of column assignments for updating existing records used in merge mode, for example `foo = foo + S.foo`. (string array, default: always overwrites with new values)
 - **ssl**: enables SSL. data will be encrypted but CA or certification will not be verified (boolean, default: false)
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)

data/src/main/java/org/embulk/output/PostgreSQLOutputPlugin.java

@@ -3,15 +3,11 @@ package org.embulk.output;
 import java.io.IOException;
 import java.sql.SQLException;
 import java.sql.Types;
-import java.util.List;
 import java.util.Properties;
 
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
-import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
-import org.embulk.output.jdbc.BatchInsert;
-import org.embulk.output.jdbc.JdbcColumn;
-import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.*;
 import org.embulk.output.jdbc.setter.ColumnSetterFactory;
 import org.embulk.output.postgresql.PostgreSQLCopyBatchInsert;
 import org.embulk.output.postgresql.PostgreSQLOutputConnector;
@@ -112,9 +108,9 @@ public class PostgreSQLOutputPlugin
     }
 
     @Override
-    protected BatchInsert newBatchInsert(PluginTask task, Optional<List<String>> mergeKeys) throws IOException, SQLException
+    protected BatchInsert newBatchInsert(PluginTask task, Optional<MergeConfig> mergeConfig) throws IOException, SQLException
     {
-        if (mergeKeys.isPresent()) {
+        if (mergeConfig.isPresent()) {
             throw new UnsupportedOperationException("PostgreSQL output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
         }
         return new PostgreSQLCopyBatchInsert(getConnector(task, true));

data/src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java

@@ -4,6 +4,8 @@ import java.util.List;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.sql.Statement;
+
+import org.embulk.output.jdbc.MergeConfig;
 import org.postgresql.copy.CopyManager;
 import org.postgresql.core.BaseConnection;
 import org.embulk.output.jdbc.JdbcOutputConnection;
@@ -27,7 +29,7 @@ public class PostgreSQLOutputConnection
         sb.append("COPY ");
         quoteIdentifierString(sb, toTable);
         sb.append(" (");
-        for (int i=0; i < toTableSchema.getCount(); i++) {
+        for (int i = 0; i < toTableSchema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, toTableSchema.getColumnName(i));
         }
@@ -43,7 +45,7 @@ public class PostgreSQLOutputConnection
     }
 
     @Override
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
@@ -51,17 +53,29 @@ public class PostgreSQLOutputConnection
         sb.append("UPDATE ");
         quoteIdentifierString(sb, toTable);
         sb.append(" SET ");
-        for (int i=0; i < schema.getCount(); i++) {
-            if (i != 0) { sb.append(", "); }
-            quoteIdentifierString(sb, schema.getColumnName(i));
-            sb.append(" = S.");
-            quoteIdentifierString(sb, schema.getColumnName(i));
+        if (mergeConfig.getMergeRule().isPresent()) {
+            List<String> rule = mergeConfig.getMergeRule().get();
+            for (int i = 0; i < rule.size(); i++) {
+                if (i != 0) {
+                    sb.append(", ");
+                }
+                sb.append(rule.get(i));
+            }
+        } else {
+            for (int i = 0; i < schema.getCount(); i++) {
+                if (i != 0) {
+                    sb.append(", ");
+                }
+                quoteIdentifierString(sb, schema.getColumnName(i));
+                sb.append(" = S.");
+                quoteIdentifierString(sb, schema.getColumnName(i));
+            }
         }
         sb.append(" FROM (");
-        for (int i=0; i < fromTables.size(); i++) {
+        for (int i = 0; i < fromTables.size(); i++) {
             if (i != 0) { sb.append(" UNION ALL "); }
             sb.append("SELECT ");
-            for(int j=0; j < schema.getCount(); j++) {
+            for(int j = 0; j < schema.getCount(); j++) {
                 if (j != 0) { sb.append(", "); }
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
@@ -70,7 +84,8 @@ public class PostgreSQLOutputConnection
         }
         sb.append(") S");
         sb.append(" WHERE ");
-        for (int i=0; i < mergeKeys.size(); i++) {
+        List<String> mergeKeys = mergeConfig.getMergeKeys();
+        for (int i = 0; i < mergeKeys.size(); i++) {
             if (i != 0) { sb.append(" AND "); }
             quoteIdentifierString(sb, toTable);
             sb.append(".");
@@ -80,7 +95,7 @@ public class PostgreSQLOutputConnection
             quoteIdentifierString(sb, mergeKeys.get(i));
         }
         sb.append(" RETURNING ");
-        for (int i=0; i < mergeKeys.size(); i++) {
+        for (int i = 0; i < mergeKeys.size(); i++) {
             if (i != 0) { sb.append(", "); }
             sb.append("S.");
             quoteIdentifierString(sb, mergeKeys.get(i));
@@ -90,21 +105,21 @@ public class PostgreSQLOutputConnection
         sb.append("INSERT INTO ");
         quoteIdentifierString(sb, toTable);
         sb.append(" (");
-        for (int i=0; i < schema.getCount(); i++) {
+        for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, schema.getColumnName(i));
         }
         sb.append(") ");
         sb.append("SELECT DISTINCT ON (");
-        for (int i=0; i < mergeKeys.size(); i++) {
+        for (int i = 0; i < mergeKeys.size(); i++) {
             if (i != 0) { sb.append(", "); }
             quoteIdentifierString(sb, mergeKeys.get(i));
         }
         sb.append(") * FROM (");
-        for (int i=0; i < fromTables.size(); i++) {
+        for (int i = 0; i < fromTables.size(); i++) {
             if (i != 0) { sb.append(" UNION ALL "); }
             sb.append("SELECT ");
-            for(int j=0; j < schema.getCount(); j++) {
+            for(int j = 0; j < schema.getCount(); j++) {
                 if (j != 0) { sb.append(", "); }
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
@@ -114,7 +129,7 @@ public class PostgreSQLOutputConnection
         sb.append(") S ");
         sb.append("WHERE NOT EXISTS (");
         sb.append("SELECT 1 FROM updated WHERE ");
-        for (int i=0; i < mergeKeys.size(); i++) {
+        for (int i = 0; i < mergeKeys.size(); i++) {
             if (i != 0) { sb.append(" AND "); }
             sb.append("S.");
             quoteIdentifierString(sb, mergeKeys.get(i));

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-postgresql
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.6.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-23 00:00:00.000000000 Z
+date: 2016-08-10 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -26,8 +26,8 @@ files:
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnection.java
 - src/main/java/org/embulk/output/postgresql/PostgreSQLOutputConnector.java
 - src/main/java/org/embulk/output/postgresql/setter/PostgreSQLColumnSetterFactory.java
-- classpath/embulk-output-jdbc-0.6.1.jar
-- classpath/embulk-output-postgresql-0.6.1.jar
+- classpath/embulk-output-jdbc-0.6.2.jar
+- classpath/embulk-output-postgresql-0.6.2.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses: