embulk-output-redshift 0.8.2 → 0.8.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f0131232d28716e555ccbed240521fd1f0cfd5bd
-  data.tar.gz: b3eccfb6c93f17f14f3640a504f8ede5ae06b9ff
+  metadata.gz: 59e6e32d3500ebab6da91b63717a6a9ea2e1eba9
+  data.tar.gz: 04a60823379e8b9c0995de2319382a5dae1b00fe
 SHA512:
-  metadata.gz: b57cf72537f09672b125b93211a5e15e4da69b45f9f404334f473c25cc82a42d093577f9733985c15597f4c90372b2d04651b622f30f5b9d63eeb663dc11482a
-  data.tar.gz: 1d08b9eb2a5b3f2dd66d6ab068038c9c13710e03fe3138f0bd19691f90dd84bfb8ee37fa3e7c7c4247e2f294338b8dd7bfd44e7ebc4ef8d2be612c8dd5b48b0a
+  metadata.gz: 80a118548222cfba3055f45c5851a227cb37aff9d6d073db172d3d3b7a1fff707584f9ab8da3102cd433b641711d6b9729a05b125670969a767248af737c05d7
+  data.tar.gz: 20905dbeb9830dc7e1bc33fbdcff0245be4abe8282a296ccb6e144d284d6c3f0dcf262bcbe20ccb008c74cb28cb6128642ea48729cbb6a85d3fcf13442d3ce43

data/README.md

@@ -21,6 +21,7 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **table**: destination table name (string, required)
 - **create_table_constraint**: table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **create_table_option**: table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **transaction_isolation**: transaction isolation level for each connection ("read_uncommitted", "read_committed", "repeatable_read" or "serializable"). if not specified, database default value will be used.
 - **access_key_id**: deprecated. `aws_access_key_id` should be used (see "basic" in `aws_auth_method`).
 - **secret_access_key**: deprecated. `aws_secret_access_key` should be used (see "basic" in `aws_auth_method`).
 - **aws_auth_method**: name of mechanism to authenticate requests ("basic", "env", "instance", "profile", "properties", "anonymous", "session" or "default". default: "basic")
@@ -74,7 +75,7 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **s3_key_prefix**: S3 key prefix for temporary files (string, default: "")
 - **delete_s3_temp_file**: whether to delete temporary files uploaded on S3 (boolean, default: true)
 - **options**: extra connection properties (hash, default: {})
-- **retry_limit**: max retry count for database operations (integer, default: 12)
+- **retry_limit**: max retry count for database operations (integer, default: 12). When intermediate table to create already created by another process, this plugin will retry with another table name to avoid collision.
 - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))
 - **max_retry_wait**: upper limit of retry wait, which will be doubled at every retry (integer, default: 1800000 (30 minutes))
 - **mode**: "insert", "insert_direct", "truncate_insert", "replace" or "merge". See below. (string, required)

data/build.gradle

@@ -2,8 +2,8 @@ dependencies {
     compile project(':embulk-output-jdbc')
     compile project(':embulk-output-postgresql')
 
-    compile "com.amazonaws:aws-java-sdk-s3:1.10.77"
-    compile "com.amazonaws:aws-java-sdk-sts:1.10.77"
+    compile "com.amazonaws:aws-java-sdk-s3:1.11.523"
+    compile "com.amazonaws:aws-java-sdk-sts:1.11.523"
     compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.21'
 
     testCompile project(':embulk-output-jdbc').sourceSets.test.output

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -16,6 +16,7 @@ import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
 import org.embulk.output.jdbc.BatchInsert;
 import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.JdbcOutputConnector;
 import org.embulk.output.jdbc.MergeConfig;
 import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.output.jdbc.Ssl;
@@ -100,7 +101,7 @@ public class RedshiftOutputPlugin
     }
 
     @Override
-    protected RedshiftOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
+    protected JdbcOutputConnector getConnector(PluginTask task, boolean retryableMetadataOperation)
     {
         RedshiftPluginTask t = (RedshiftPluginTask) task;
 
@@ -140,7 +141,7 @@ public class RedshiftOutputPlugin
         props.setProperty("password", t.getPassword());
         logConnectionProperties(url, props);
 
-        return new RedshiftOutputConnector(url, props, t.getSchema());
+        return new RedshiftOutputConnector(url, props, t.getSchema(), t.getTransactionIsolation());
     }
 
     private static AWSCredentialsProvider getAWSCredentialsProvider(RedshiftPluginTask task)

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -17,6 +17,7 @@ import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.zip.GZIPOutputStream;
 
+import org.embulk.output.jdbc.JdbcOutputConnector;
 import org.embulk.output.jdbc.JdbcSchema;
 import org.embulk.output.jdbc.TableIdentifier;
 import org.embulk.output.postgresql.AbstractPostgreSQLCopyBatchInsert;
@@ -42,7 +43,7 @@ public class RedshiftCopyBatchInsert
         extends AbstractPostgreSQLCopyBatchInsert
 {
     private final Logger logger = Exec.getLogger(RedshiftCopyBatchInsert.class);
-    private final RedshiftOutputConnector connector;
+    private final JdbcOutputConnector connector;
     private final String s3BucketName;
     private final String s3KeyPrefix;
     private final String iamReaderUserName;
@@ -61,7 +62,7 @@ public class RedshiftCopyBatchInsert
 
     public static final String COPY_AFTER_FROM = "GZIP DELIMITER '\\t' NULL '\\\\N' ESCAPE TRUNCATECOLUMNS ACCEPTINVCHARS STATUPDATE OFF COMPUPDATE OFF";
 
-    public RedshiftCopyBatchInsert(RedshiftOutputConnector connector,
+    public RedshiftCopyBatchInsert(JdbcOutputConnector connector,
             AWSCredentialsProvider credentialsProvider, String s3BucketName, String s3KeyPrefix,
             String iamReaderUserName, boolean deleteS3TempFile) throws IOException, SQLException
     {
@@ -98,7 +99,7 @@ public class RedshiftCopyBatchInsert
     @Override
     public void prepare(TableIdentifier loadTable, JdbcSchema insertSchema) throws SQLException
     {
-        this.connection = connector.connect(true);
+        this.connection = (RedshiftOutputConnection)connector.connect(true);
         this.copySqlBeforeFrom = connection.buildCopySQLBeforeFrom(loadTable, insertSchema);
         logger.info("Copy SQL: "+copySqlBeforeFrom+" ? "+COPY_AFTER_FROM);
     }
@@ -137,8 +138,6 @@ public class RedshiftCopyBatchInsert
     @Override
     public void finish() throws IOException, SQLException
     {
-        super.finish();
-
         for (Future<Void> uploadAndCopyFuture : uploadAndCopyFutures) {
             try {
                 uploadAndCopyFuture.get();
@@ -269,7 +268,7 @@ public class RedshiftCopyBatchInsert
             try {
                 uploadFuture.get();
 
-                RedshiftOutputConnection con = connector.connect(true);
+                RedshiftOutputConnection con = (RedshiftOutputConnection)connector.connect(true);
                 try {
                     logger.info("Running COPY from file {}", s3KeyName);
 

data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java

@@ -19,11 +19,10 @@ public class RedshiftOutputConnection
 {
     private final Logger logger = Exec.getLogger(RedshiftOutputConnection.class);
 
-    public RedshiftOutputConnection(Connection connection, String schemaName, boolean autoCommit)
+    public RedshiftOutputConnection(Connection connection, String schemaName)
             throws SQLException
     {
         super(connection, schemaName);
-        connection.setAutoCommit(autoCommit);
     }
 
     // ALTER TABLE cannot change the schema of a table

data/src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java

@@ -4,11 +4,15 @@ import java.util.Properties;
 import java.sql.Driver;
 import java.sql.Connection;
 import java.sql.SQLException;
-import org.embulk.output.jdbc.JdbcOutputConnector;
+
+import org.embulk.output.jdbc.AbstractJdbcOutputConnector;
 import org.embulk.output.jdbc.JdbcOutputConnection;
+import org.embulk.output.jdbc.TransactionIsolation;
+
+import com.google.common.base.Optional;
 
 public class RedshiftOutputConnector
-        implements JdbcOutputConnector
+        extends AbstractJdbcOutputConnector
 {
     private static final Driver driver = new org.postgresql.Driver();
 
@@ -16,19 +20,21 @@ public class RedshiftOutputConnector
     private final Properties properties;
     private final String schemaName;
 
-    public RedshiftOutputConnector(String url, Properties properties, String schemaName)
+    public RedshiftOutputConnector(String url, Properties properties, String schemaName,
+            Optional<TransactionIsolation> transactionIsolation)
     {
+        super(transactionIsolation);
         this.url = url;
         this.properties = properties;
         this.schemaName = schemaName;
     }
 
     @Override
-    public RedshiftOutputConnection connect(boolean autoCommit) throws SQLException
+    protected JdbcOutputConnection connect() throws SQLException
     {
         Connection c = driver.connect(url, properties);
         try {
-            RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName, autoCommit);
+            RedshiftOutputConnection con = new RedshiftOutputConnection(c, schemaName);
             c = null;
             return con;
         } finally {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.8.2
+  version: 0.8.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-01-30 00:00:00.000000000 Z
+date: 2019-05-07 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,23 +19,23 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- classpath/aws-java-sdk-core-1.11.253.jar
-- classpath/aws-java-sdk-kms-1.11.253.jar
-- classpath/aws-java-sdk-s3-1.11.253.jar
-- classpath/aws-java-sdk-sts-1.10.77.jar
-- classpath/commons-codec-1.9.jar
+- classpath/aws-java-sdk-core-1.11.523.jar
+- classpath/aws-java-sdk-kms-1.11.523.jar
+- classpath/aws-java-sdk-s3-1.11.523.jar
+- classpath/aws-java-sdk-sts-1.11.523.jar
+- classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-output-jdbc-0.8.2.jar
-- classpath/embulk-output-postgresql-0.8.2.jar
-- classpath/embulk-output-redshift-0.8.2.jar
+- classpath/embulk-output-jdbc-0.8.3.jar
+- classpath/embulk-output-postgresql-0.8.3.jar
+- classpath/embulk-output-redshift-0.8.3.jar
 - classpath/embulk-util-aws-credentials-0.2.21.jar
-- classpath/httpclient-4.5.2.jar
-- classpath/httpcore-4.4.4.jar
+- classpath/httpclient-4.5.5.jar
+- classpath/httpcore-4.4.9.jar
 - classpath/ion-java-1.0.2.jar
-- classpath/jackson-databind-2.6.7.1.jar
+- classpath/jackson-databind-2.6.7.2.jar
 - classpath/jackson-dataformat-cbor-2.6.7.jar
 - classpath/jcl-over-slf4j-1.7.12.jar
-- classpath/jmespath-java-1.11.253.jar
+- classpath/jmespath-java-1.11.523.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/output/redshift.rb
 - src/main/java/org/embulk/output/RedshiftOutputPlugin.java