embulk-output-redshift 0.8.5 → 0.8.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 92f693ebcc86c0b3e1c0e104df54225d136fe814
-  data.tar.gz: e8a8178f1f136a4bcd675db98c8f8aebd4fbcae0
+  metadata.gz: 3b6636ae2a50de2f8889c15227a842a2712261f8
+  data.tar.gz: bb5ce05600900bea03ad46e00825aa7cdf088390
 SHA512:
-  metadata.gz: 0be7bcfdbd1e47fe6868dc734fc5f75875f11889a1c4db206d753fd8be2ebb6735531bd7b04e7cd0f37ed2420e57bf8817cbc684e0596dc5f8f79a7f793be336
-  data.tar.gz: f37d86eb0ae27ba418f6dc13ee56b841ee747fa212eb88b09e45cf1c1836c9b59f13f2bfeca5fe60782bda3280e3a718caebf83a0c48c185e88b888d1f6570d1
+  metadata.gz: 1056dc021dd92ab4486ed4d065607e910fdb551b9a10e018cb14cd65f3b56ea84e246aeeee7fd5d58eec7474a9f61762f785da51a40e7f1c2c70b35b01e70661
+  data.tar.gz: e13333d0458a8021a92248bb4fad097f1e5c391061fad94c9d0beb4df2b3d88c15468562f2e8e07988b38fb48be5cc4d76571b5d8f932c38b424bd3266e9f1f8

data/README.md

@@ -74,6 +74,8 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **s3_bucket**: S3 bucket name for temporary files
 - **s3_key_prefix**: S3 key prefix for temporary files (string, default: "")
 - **delete_s3_temp_file**: whether to delete temporary files uploaded on S3 (boolean, default: true)
+- **copy_iam_role_name**: IAM Role for COPY credential(https://docs.aws.amazon.com/redshift/latest/dg/copy-usage_notes-access-permissions.html), if this is set, IAM Role is used instead of aws access key and aws secret access key(string, optional)
+- **copy_aws_account_id**: IAM Role's account ID for multi account COPY. If this is set, the ID is used instead of authenticated user's account ID. This is enabled only if copy_iam_role_name is set.(string, optional)
 - **options**: extra connection properties (hash, default: {})
 - **retry_limit**: max retry count for database operations (integer, default: 12). When intermediate table to create already created by another process, this plugin will retry with another table name to avoid collision.
 - **retry_wait**: initial retry wait time in milliseconds (integer, default: 1000 (1 second))

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -83,6 +83,14 @@ public class RedshiftOutputPlugin
         @Config("ssl")
         @ConfigDefault("\"disable\"")
         public Ssl getSsl();
+
+        @Config("copy_iam_role_name")
+        @ConfigDefault("null")
+        public Optional<String> getCopyIamRoleName();
+
+        @Config("copy_aws_account_id")
+        @ConfigDefault("null")
+        public Optional<String> getCopyAwsAccountId();
     }
 
     @Override
@@ -193,6 +201,6 @@ public class RedshiftOutputPlugin
         RedshiftPluginTask t = (RedshiftPluginTask) task;
         setAWSCredentialsBackwardCompatibility(t);
         return new RedshiftCopyBatchInsert(getConnector(task, true),
-                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName(), t.getDeleteS3TempFile());
+                getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName(), t.getDeleteS3TempFile(), t.getCopyIamRoleName().orNull(), t.getCopyAwsAccountId().orNull());
     }
 }

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -36,6 +36,8 @@ import com.amazonaws.services.s3.AmazonS3Client;
 import com.amazonaws.services.s3.model.Region;
 import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
 import com.amazonaws.services.securitytoken.model.Credentials;
+import com.amazonaws.services.securitytoken.model.GetCallerIdentityRequest;
+import com.amazonaws.services.securitytoken.model.GetCallerIdentityResult;
 import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
 import com.amazonaws.services.securitytoken.model.GetFederationTokenResult;
 
@@ -53,6 +55,7 @@ public class RedshiftCopyBatchInsert
     private final String s3RegionName;
     private final AWSSecurityTokenServiceClient sts;
     private final ExecutorService executorService;
+    private final String copyIamRoleARN;
 
     private RedshiftOutputConnection connection = null;
     private String copySqlBeforeFrom = null;
@@ -64,7 +67,7 @@ public class RedshiftCopyBatchInsert
 
     public RedshiftCopyBatchInsert(JdbcOutputConnector connector,
             AWSCredentialsProvider credentialsProvider, String s3BucketName, String s3KeyPrefix,
-            String iamReaderUserName, boolean deleteS3TempFile) throws IOException, SQLException
+            String iamReaderUserName, boolean deleteS3TempFile, String copyIamRoleName, String copyAwsAccountId) throws IOException, SQLException
     {
         super();
         this.connector = connector;
@@ -94,6 +97,20 @@ public class RedshiftCopyBatchInsert
                     + " IAM user needs \"s3:GetBucketLocation\" permission if Redshift region and S3 region are different.");
         }
         this.s3RegionName = s3RegionName;
+
+        if (copyIamRoleName != null) {
+            String accountId = null;
+            if (copyAwsAccountId != null) {
+                accountId = copyAwsAccountId;
+            } else {
+                GetCallerIdentityRequest request = new GetCallerIdentityRequest();
+                GetCallerIdentityResult response = this.sts.getCallerIdentity(request);
+                accountId = response.getAccount();
+            }
+            this.copyIamRoleARN = "arn:aws:iam::" + accountId + ":role/" + copyIamRoleName;
+        } else {
+            this.copyIamRoleARN = null;
+        }
     }
 
     @Override
@@ -305,13 +322,18 @@ public class RedshiftCopyBatchInsert
             sb.append("/");
             sb.append(s3KeyName);
             sb.append("' CREDENTIALS '");
-            sb.append("aws_access_key_id=");
-            sb.append(creds.getAWSAccessKeyId());
-            sb.append(";aws_secret_access_key=");
-            sb.append(creds.getAWSSecretKey());
-            if (creds.getSessionToken() != null) {
-                sb.append(";token=");
-                sb.append(creds.getSessionToken());
+            if (copyIamRoleARN != null) {
+                sb.append("aws_iam_role=");
+                sb.append(copyIamRoleARN);
+            } else {
+                sb.append("aws_access_key_id=");
+                sb.append(creds.getAWSAccessKeyId());
+                sb.append(";aws_secret_access_key=");
+                sb.append(creds.getAWSSecretKey());
+                if (creds.getSessionToken() != null) {
+                    sb.append(";token=");
+                    sb.append(creds.getSessionToken());
+                }
             }
             sb.append("' ");
             if (s3RegionName != null) {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.8.5
+  version: 0.8.6
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-05-10 00:00:00.000000000 Z
+date: 2019-08-22 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -25,9 +25,9 @@ files:
 - classpath/aws-java-sdk-sts-1.11.523.jar
 - classpath/commons-codec-1.10.jar
 - classpath/commons-logging-1.2.jar
-- classpath/embulk-output-jdbc-0.8.5.jar
-- classpath/embulk-output-postgresql-0.8.5.jar
-- classpath/embulk-output-redshift-0.8.5.jar
+- classpath/embulk-output-jdbc-0.8.6.jar
+- classpath/embulk-output-postgresql-0.8.6.jar
+- classpath/embulk-output-redshift-0.8.6.jar
 - classpath/embulk-util-aws-credentials-0.2.21.jar
 - classpath/httpclient-4.5.5.jar
 - classpath/httpcore-4.4.9.jar