embulk-output-redshift 0.6.0 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 060e56b3c63aab3ac5b3e8f203b3641858b7be38
-  data.tar.gz: 16c1812e630d1a6f63e537e504976a977a5fd0bc
+  metadata.gz: 4befc609ebdd8278a0dc58d63c464d710bde3ed7
+  data.tar.gz: d65e65a04861aba8e25b33d3ccb5ae6f0a406bc9
 SHA512:
-  metadata.gz: f698fcecef53b3f6cd2e6e4436531621c0c84eead54796b52d85f40706ebc6b2e30dad0ebfa46b10fd4a72d14ef570dc36222fa00f85549898a43da312f03601
-  data.tar.gz: 52d423812bad7448cf85e4e6c0632744e9cb23ff2e248d6101d82c2b60cbe5b19668bae2d1415a747428173e132917afd2fb77f9cf004e76c414c5438c8ba1c9
+  metadata.gz: 5675fe7972621547f61caa17f303461c98c5bee60f5354f06ae42ae0e1546a22ccf1ae339b08436fc1b49ac9022228f8192ccdcfbbe956d50ff74a7cbfc45b56
+  data.tar.gz: 63b40e72eed84caa98c178fefa3e603ef7ee2661af0d65126ec9ce38181a428a7072ce6fe6fe5995c7f3643ca539b7aba7138dd81b06497b3ffa5230b5cb0bcb

data/README.md

@@ -23,7 +23,8 @@ Redshift output plugins for Embulk loads records to Redshift.
 - **s3_bucket**: S3 bucket name for temporary files
 - **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
 - **options**: extra connection properties (hash, default: {})
-- **mode**: "insert", "insert_direct", "truncate_insert", or "replace". See below. (string, required)
+- **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)
+- **merge_keys**: key column names for merging records in merge mode (string array, required in merge mode)
 - **batch_size**: size of a single batch insert (integer, default: 16777216)
 - **default_timezone**: If input column type (embulk type) is timestamp, this plugin needs to format the timestamp into a SQL string. This default_timezone option is used to control the timezone. You can overwrite timezone for each columns using column_options option. (string, default: `UTC`)
 - **column_options**: advanced: a key-value pairs where key is a column name and value is options for the column.
@@ -51,6 +52,10 @@ Redshift output plugins for Embulk loads records to Redshift.
   * Behavior: This mode writes rows to an intermediate table first. If all those tasks run correctly, drops the target table and alters the name of the intermediate table into the target table name.
   * Transactional: Yes.
   * Resumable: No.
+* **merge**:
+  * Behavior: This mode writes rows to some intermediate tables first. If all those tasks run correctly, inserts records from intermediate tables after deleting records whose keys exist in intermediate tables. Namely, if merge keys of a record in the intermediate tables already exist in the target table, the target record is updated by the intermediate record, otherwise the intermediate record is inserted. If the target table doesn't exist, it is created automatically.
+  * Transactional: Yes.
+  * Resumable: Yes.
 
 ### Supported types
 

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -75,7 +75,7 @@ public class RedshiftOutputPlugin
     {
         return new Features()
             .setMaxTableNameLength(30)
-            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE))
+            .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE, Mode.MERGE))
             .setIgnoreMergeKeys(false);
     }
 

data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java

@@ -1,5 +1,6 @@
 package org.embulk.output.redshift;
 
+import java.util.List;
 import java.sql.Connection;
 import java.sql.SQLException;
 import java.sql.Statement;
@@ -119,4 +120,52 @@ public class RedshiftOutputConnection
             stmt.close();
         }
     }
+
+    @Override
+    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, List<String> mergeKeys) throws SQLException
+    {
+        StringBuilder sb = new StringBuilder();
+
+        sb.append("BEGIN TRANSACTION;");
+
+        sb.append("DELETE FROM ");
+        quoteIdentifierString(sb, toTable);
+        sb.append(" USING (");
+        for(int i=0; i < fromTables.size(); i++) {
+            if(i != 0) { sb.append(" UNION ALL "); }
+            sb.append("SELECT * FROM ");
+            quoteIdentifierString(sb, fromTables.get(i));
+        }
+        sb.append(") S WHERE (");
+        for(int i=0; i < mergeKeys.size(); i++) {
+            if (i != 0) { sb.append(" AND "); }
+            sb.append("S.");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+            sb.append(" = ");
+            quoteIdentifierString(sb, toTable);
+            sb.append(".");
+            quoteIdentifierString(sb, mergeKeys.get(i));
+        }
+        sb.append(");");
+
+        sb.append("INSERT INTO ");
+        quoteIdentifierString(sb, toTable);
+        sb.append(" (");
+        for(int i=0; i < fromTables.size(); i++) {
+            if(i != 0) { sb.append(" UNION ALL "); }
+            sb.append("SELECT ");
+            for(int j=0; j < schema.getCount(); j++) {
+                if (j != 0) { sb.append(", "); }
+                quoteIdentifierString(sb, schema.getColumnName(j));
+            }
+            sb.append(" FROM ");
+            quoteIdentifierString(sb, fromTables.get(i));
+        }
+        sb.append(");");
+
+        sb.append("END TRANSACTION;");
+
+        return sb.toString();
+    }
+
 }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.6.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-04-26 00:00:00.000000000 Z
+date: 2016-06-23 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -30,9 +30,9 @@ files:
 - classpath/aws-java-sdk-sts-1.10.33.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-jdbc-0.6.0.jar
-- classpath/embulk-output-postgresql-0.6.0.jar
-- classpath/embulk-output-redshift-0.6.0.jar
+- classpath/embulk-output-jdbc-0.6.1.jar
+- classpath/embulk-output-postgresql-0.6.1.jar
+- classpath/embulk-output-redshift-0.6.1.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar