embulk-output-redshift 0.7.8 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d7fe1d8967d1047d320f4f97db4c9877d380dd9
-  data.tar.gz: 500599fe7553ef8178d4b770d1cf18ee94d4c0a8
+  metadata.gz: 36ac2e6be278272eb350e37f52c533a7db3da42c
+  data.tar.gz: dca6057f28230550806b00e716ec128be9df17ca
 SHA512:
-  metadata.gz: 834967f091fdc5bbe1747936096bcc450ca3b190fd0d96b2c325016999aedf97b73021ea97804f63139dc5184e32a83310d76a8d21c757e5de4c8e11c27584e0
-  data.tar.gz: 1721e9f0a10559c8a432736f4af94ce1f040a5925246c8fdfa145601acb03471ba8e472788f11cede92748dbe8304dec9477083717676fed3e11530d09db0152
+  metadata.gz: 43141c1afaabca1dede02262d2b796ed073b38bf9a2124fbc7d30aed5a50a23cb7ec092ca9f1d15717e0d2f4d267a7144fdf879c72ce738a5cde9ce26f02a25e
+  data.tar.gz: 912553f1443a79805b90f9a57c4ed01e7ddc1115eef21259ed96b76ac8129c3b91b0bc29fe5b71f840f4756a9c0548b445cbcd7d791c1593ccc6307dea1a429b

data/README.md

@@ -17,6 +17,7 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **password**: database login password (string, default: "")
 - **database**: destination database name (string, required)
 - **schema**: destination schema name (string, default: "public")
+- **temp_schema**: schema name for intermediate tables. by default, intermediate tables will be created in the schema specified by `schema`. replace mode doesn't support temp_schema. (string, optional)
 - **table**: destination table name (string, required)
 - **access_key_id**: deprecated. `aws_access_key_id` should be used (see "basic" in `aws_auth_method`).
 - **secret_access_key**: deprecated. `aws_secret_access_key` should be used (see "basic" in `aws_auth_method`).

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -4,7 +4,6 @@ import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
 
-import org.embulk.output.jdbc.MergeConfig;
 import org.slf4j.Logger;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
@@ -17,6 +16,8 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.output.redshift.RedshiftOutputConnector;
 import org.embulk.output.redshift.RedshiftCopyBatchInsert;
 import org.embulk.output.redshift.Ssl;
@@ -49,6 +50,10 @@ public class RedshiftOutputPlugin
         @ConfigDefault("\"public\"")
         public String getSchema();
 
+        @Config("temp_schema")
+        @ConfigDefault("null")
+        public Optional<String> getTempSchema();
+
         // for backward compatibility
         @Config("access_key_id")
         @ConfigDefault("null")
@@ -153,6 +158,16 @@ public class RedshiftOutputPlugin
         }
     }
 
+    @Override
+    protected TableIdentifier buildIntermediateTableId(JdbcOutputConnection con, PluginTask task, String tableName) {
+        RedshiftPluginTask t = (RedshiftPluginTask) task;
+        // replace mode doesn't support temp_schema because ALTER TABLE cannot change schema of table
+        if (t.getTempSchema().isPresent() && t.getMode() != Mode.REPLACE) {
+            return new TableIdentifier(null, t.getTempSchema().get(), tableName);
+        }
+        return super.buildIntermediateTableId(con, task, tableName);
+    }
+
     @Override
     protected String generateIntermediateTableNamePrefix(String baseTableName, JdbcOutputConnection con,
             int suffixLength, int maxLength, LengthSemantics lengthSemantics) throws SQLException {

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -18,6 +18,7 @@ import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPOutputStream;
 
 import org.embulk.output.jdbc.JdbcSchema;
+import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.output.postgresql.AbstractPostgreSQLCopyBatchInsert;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
@@ -93,7 +94,7 @@ public class RedshiftCopyBatchInsert
     }
 
     @Override
-    public void prepare(String loadTable, JdbcSchema insertSchema) throws SQLException
+    public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
     {
         this.connection = connector.connect(true);
         this.copySqlBeforeFrom = connection.buildCopySQLBeforeFrom(loadTable, insertSchema);

data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java

@@ -10,6 +10,7 @@ import org.embulk.output.jdbc.JdbcColumn;
 import org.embulk.output.jdbc.JdbcOutputConnection;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.MergeConfig;
+import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
@@ -30,11 +31,11 @@ public class RedshiftOutputConnection
     // Redshift does not support DROP TABLE IF EXISTS.
     // Here runs DROP TABLE and ignores errors.
     @Override
-    public void dropTableIfExists(String tableName) throws SQLException
+    public void dropTableIfExists(TableIdentifier table) throws SQLException
     {
         Statement stmt = connection.createStatement();
         try {
-            String sql = String.format("DROP TABLE IF EXISTS %s", quoteIdentifierString(tableName));
+            String sql = String.format("DROP TABLE IF EXISTS %s", quoteTableIdentifier(table));
             executeUpdate(stmt, sql);
             commitIfNecessary(connection);
         } catch (SQLException ex) {
@@ -49,14 +50,14 @@ public class RedshiftOutputConnection
     // Redshift does not support DROP TABLE IF EXISTS.
     // Dropping part runs DROP TABLE and ignores errors.
     @Override
-    public void replaceTable(String fromTable, JdbcSchema schema, String toTable, Optional<String> additionalSql) throws SQLException
+    public void replaceTable(TableIdentifier fromTable, JdbcSchema schema, TableIdentifier toTable, Optional<String> additionalSql) throws SQLException
     {
         Statement stmt = connection.createStatement();
         try {
             try {
                 StringBuilder sb = new StringBuilder();
                 sb.append("DROP TABLE ");
-                quoteIdentifierString(sb, toTable);
+                quoteTableIdentifier(sb, toTable);
                 String sql = sb.toString();
                 executeUpdate(stmt, sql);
             } catch (SQLException ex) {
@@ -67,11 +68,12 @@ public class RedshiftOutputConnection
             }
 
             {
+                // ALTER TABLE cannot change schema of table
                 StringBuilder sb = new StringBuilder();
                 sb.append("ALTER TABLE ");
-                quoteIdentifierString(sb, fromTable);
+                quoteTableIdentifier(sb, fromTable);
                 sb.append(" RENAME TO ");
-                quoteIdentifierString(sb, toTable);
+                quoteIdentifierString(sb, toTable.getTableName());
                 String sql = sb.toString();
                 executeUpdate(stmt, sql);
             }
@@ -104,12 +106,12 @@ public class RedshiftOutputConnection
         }
     }
 
-    public String buildCopySQLBeforeFrom(String tableName, JdbcSchema tableSchema)
+    public String buildCopySQLBeforeFrom(TableIdentifier table, JdbcSchema tableSchema)
     {
         StringBuilder sb = new StringBuilder();
 
         sb.append("COPY ");
-        quoteIdentifierString(sb, tableName);
+        quoteTableIdentifier(sb, table);
         sb.append(" (");
         for(int i=0; i < tableSchema.getCount(); i++) {
             if(i != 0) { sb.append(", "); }
@@ -131,7 +133,7 @@ public class RedshiftOutputConnection
     }
 
     @Override
-    protected String buildCollectMergeSql(List<String> fromTables, JdbcSchema schema, String toTable, MergeConfig mergeConfig) throws SQLException
+    protected String buildCollectMergeSql(List<TableIdentifier> fromTables, JdbcSchema schema, TableIdentifier toTable, MergeConfig mergeConfig) throws SQLException
     {
         StringBuilder sb = new StringBuilder();
 
@@ -148,7 +150,7 @@ public class RedshiftOutputConnection
         sb.append("BEGIN TRANSACTION;");
 
         sb.append("UPDATE ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" SET ");
         for (int i = 0; i < updateKeys.size(); i++) {
             if (i != 0) { sb.append(", "); }
@@ -166,7 +168,7 @@ public class RedshiftOutputConnection
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
             sb.append(" FROM ");
-            quoteIdentifierString(sb, fromTables.get(i));
+            quoteTableIdentifier(sb, fromTables.get(i));
         }
         sb.append(" ) S WHERE ");
 
@@ -175,14 +177,14 @@ public class RedshiftOutputConnection
             sb.append("S.");
             quoteIdentifierString(sb, mergeKeys.get(i));
             sb.append(" = ");
-            quoteIdentifierString(sb, toTable);
+            quoteTableIdentifier(sb, toTable);
             sb.append(".");
             quoteIdentifierString(sb, mergeKeys.get(i));
         }
         sb.append(";");
 
         sb.append("INSERT INTO ");
-        quoteIdentifierString(sb, toTable);
+        quoteTableIdentifier(sb, toTable);
         sb.append(" (");
         for (int i = 0; i < schema.getCount(); i++) {
             if (i != 0) { sb.append(", "); }
@@ -197,18 +199,20 @@ public class RedshiftOutputConnection
                 quoteIdentifierString(sb, schema.getColumnName(j));
             }
             sb.append(" FROM ");
-            quoteIdentifierString(sb, fromTables.get(i));
+            quoteTableIdentifier(sb, fromTables.get(i));
             sb.append(" WHERE NOT EXISTS (SELECT 1 FROM ");
-            quoteIdentifierString(sb, toTable);
+            quoteTableIdentifier(sb, toTable);
+            sb.append(", ");
+            quoteTableIdentifier(sb, fromTables.get(i));
             sb.append(" WHERE ");
 
             for (int k = 0; k < mergeKeys.size(); k++) {
                 if (k != 0) { sb.append(" AND "); }
-                quoteIdentifierString(sb, fromTables.get(i));
+                quoteTableIdentifier(sb, fromTables.get(i));
                 sb.append(".");
                 quoteIdentifierString(sb, mergeKeys.get(k));
                 sb.append(" = ");
-                quoteIdentifierString(sb, toTable);
+                quoteTableIdentifier(sb, toTable);
                 sb.append(".");
                 quoteIdentifierString(sb, mergeKeys.get(k));
             }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.7.8
+  version: 0.7.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-05-10 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -26,9 +26,9 @@ files:
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
 - classpath/embulk-core-0.8.9.jar
-- classpath/embulk-output-jdbc-0.7.8.jar
-- classpath/embulk-output-postgresql-0.7.8.jar
-- classpath/embulk-output-redshift-0.7.8.jar
+- classpath/embulk-output-jdbc-0.7.9.jar
+- classpath/embulk-output-postgresql-0.7.9.jar
+- classpath/embulk-output-redshift-0.7.9.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar