embulk-output-redshift 0.7.12 → 0.7.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f49d2ce2f0a28ab1a34ad8e0a1a5b3df3dac5e8
-  data.tar.gz: da8446b33c11ac4f1736d37ad60c9e3dd9faf2f8
+  metadata.gz: 306629e8d5f5a0a452aa9e341d0e564f4d7599f7
+  data.tar.gz: dc5fef08ee347b5e72065a051d6769b16bbab7cf
 SHA512:
-  metadata.gz: f6f4a3167321def1c531d9e94f1988441309a9127949f508b660eae5a6dad5763e52c434681bf670f06026bf6aed8babfccb10f3193115468d06f639cb203d91
-  data.tar.gz: 8f7681e29c631b46570f87d43920f64f3b6e5c8440becdaf48aa5d71440acd9031a8cb227b78ab442fe1c18c33cea061fe580d8f15fd386874a930e150193b4e
+  metadata.gz: 36ed60b43124aae3ab0352c627e4fb54ddf103c943d39bc1db619112a9f0e5054489fbb390cc4ee67c3f5a234059ebba41be27311cb9ea9ce5bce11d69f1ec8b
+  data.tar.gz: baf054ea4d486e697f93b70f2304f5b59ff46c18ed97d34200922de099434e85a86a29f17884c5a64646a9e5c9a9e3441c450fc72b0943c2129e3c20989ad8d9

data/README.md

@@ -19,6 +19,8 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **schema**: destination schema name (string, default: "public")
 - **temp_schema**: schema name for intermediate tables. by default, intermediate tables will be created in the schema specified by `schema`. replace mode doesn't support temp_schema. (string, optional)
 - **table**: destination table name (string, required)
+- **create_table_constraint** table constraint added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
+- **create_table_option** table option added to `CREATE TABLE` statement, like `CREATE TABLE <table_name> (<column1> <type1>, <column2> <type2>, ..., <create_table_constraint>) <create_table_option>`.
 - **access_key_id**: deprecated. `aws_access_key_id` should be used (see "basic" in `aws_auth_method`).
 - **secret_access_key**: deprecated. `aws_secret_access_key` should be used (see "basic" in `aws_auth_method`).
 - **aws_auth_method**: name of mechanism to authenticate requests ("basic", "env", "instance", "profile", "properties", "anonymous", or "session". default: "basic")
@@ -62,7 +64,8 @@ Redshift output plugin for Embulk loads records to Redshift.
 
 - **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` and `sts:GetFederationToken`. And furthermore, the user should have permission of `s3:GetBucketLocation` if Redshift region and S3 bucket region are different. (string, default: "", but we strongly recommend that you use IAM user for security reasons. see below.)
 - **s3_bucket**: S3 bucket name for temporary files
-- **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
+- **s3_key_prefix**: S3 key prefix for temporary files (string, default: "")
+- **delete_s3_temp_file**: whether to delete temporary files uploaded on S3 (boolean, default: true)
 - **options**: extra connection properties (hash, default: {})
 - **retry_limit** max retry count for database operations (integer, default: 12)
 - **retry_wait** initial retry wait time in milliseconds (integer, default: 1000 (1 second))

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -75,6 +75,10 @@ public class RedshiftOutputPlugin
         @ConfigDefault("\"\"")
         public String getS3KeyPrefix();
 
+        @Config("delete_s3_temp_file")
+        @ConfigDefault("true")
+        public boolean getDeleteS3TempFile();
+
         @Config("ssl")
         @ConfigDefault("\"disable\"")
         public Ssl getSsl();
@@ -90,7 +94,7 @@ public class RedshiftOutputPlugin
     protected Features getFeatures(PluginTask task)
     {
         return new Features()
-            .setMaxTableNameLength(30)
+            .setMaxTableNameLength(127)
             .setSupportedModes(ImmutableSet.of(Mode.INSERT, Mode.INSERT_DIRECT, Mode.TRUNCATE_INSERT, Mode.REPLACE, Mode.MERGE))
             .setIgnoreMergeKeys(false);
     }
@@ -188,6 +192,6 @@ public class RedshiftOutputPlugin
         RedshiftPluginTask t = (RedshiftPluginTask) task;
         setAWSCredentialsBackwardCompatibility(t);
         return new RedshiftCopyBatchInsert(getConnector(task, true),
-                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName());
+                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName(), t.getDeleteS3TempFile());
     }
 }

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -46,6 +46,7 @@ public class RedshiftCopyBatchInsert
     private final String s3BucketName;
     private final String s3KeyPrefix;
     private final String iamReaderUserName;
+    private final boolean deleteS3TempFile;
     private final AWSCredentialsProvider credentialsProvider;
     private final AmazonS3Client s3;
     private final String s3RegionName;
@@ -62,7 +63,7 @@ public class RedshiftCopyBatchInsert
 
     public RedshiftCopyBatchInsert(RedshiftOutputConnector connector,
             AWSCredentialsProvider credentialsProvider, String s3BucketName, String s3KeyPrefix,
-            String iamReaderUserName) throws IOException, SQLException
+            String iamReaderUserName, boolean deleteS3TempFile) throws IOException, SQLException
     {
         super();
         this.connector = connector;
@@ -73,6 +74,7 @@ public class RedshiftCopyBatchInsert
             this.s3KeyPrefix = s3KeyPrefix + "/";
         }
         this.iamReaderUserName = iamReaderUserName;
+        this.deleteS3TempFile = deleteS3TempFile;
         this.credentialsProvider = credentialsProvider;
         this.s3 = new AmazonS3Client(credentialsProvider);  // TODO options
         this.sts = new AWSSecurityTokenServiceClient(credentialsProvider);  // options
@@ -268,7 +270,9 @@ public class RedshiftCopyBatchInsert
                     con.close();
                 }
             } finally {
-                s3.deleteObject(s3BucketName, s3KeyName);
+                if (deleteS3TempFile) {
+                    s3.deleteObject(s3BucketName, s3KeyName);
+                }
             }
 
             return null;

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.7.12
+  version: 0.7.13
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-24 00:00:00.000000000 Z
+date: 2017-12-08 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -25,9 +25,9 @@ files:
 - classpath/aws-java-sdk-sts-1.10.77.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-jdbc-0.7.12.jar
-- classpath/embulk-output-postgresql-0.7.12.jar
-- classpath/embulk-output-redshift-0.7.12.jar
+- classpath/embulk-output-jdbc-0.7.13.jar
+- classpath/embulk-output-postgresql-0.7.13.jar
+- classpath/embulk-output-redshift-0.7.13.jar
 - classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar