embulk-output-redshift 0.6.3 → 0.6.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3deb8e2a10b3e5c04d32c8a286b77a7c20364fd3
-  data.tar.gz: 6c0fbe7df77c8d10c0cdc486984dd20e05dbc55e
+  metadata.gz: 4f9b1303b4c1343a10b77b15b07fca1536c919ac
+  data.tar.gz: 3fe7932bc6a6a3d0125c88e9e60c8e9ac05cb4d6
 SHA512:
-  metadata.gz: 134476b326c5b3abfb332f92fece008a6c0dde1164ebcf8b9598e459d79302b41b869f5a4177c330cd12c105cae7585990df12a6c796852c77478191b7d31985
-  data.tar.gz: 5b443bb67b4c079e64aa4966a6c2e27bb07c89f30ae042a442d9b287385fa20fb40f45149f2a09b2ee73a193e930c8d355bacc2313dcd2f392d37b4d2358d132
+  metadata.gz: e06bf310d402011178bb190ae8ad2240ad97ddebba96e54f681e49890e52de6cb5175c6a78152bb124a70159a345a21ad5aa67745e145a34116598736f4e28bf
+  data.tar.gz: 434386a4927d5f7df605142e93c5870ec3c7b3194bac5c7d3b0b56161c51beec09f44480510dcbe290412a3846c125874caa62b95a9651ac1631738a457eaad0

data/README.md

@@ -20,7 +20,7 @@ Redshift output plugins for Embulk loads records to Redshift.
 - **table**: destination table name (string, required)
 - **access_key_id**: access key id for AWS
 - **secret_access_key**: secret access key for AWS
-- **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, , `s3:ListBucket` and `sts:GetFederationToken`. (string, default: "", but we strongly recommend that you use IAM user for security reasons. see below.)
+- **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` and `sts:GetFederationToken`. And furthermore, the user should have permission of `s3:GetBucketLocation` if Redshift region and S3 bucket region are different. (string, default: "", but we strongly recommend that you use IAM user for security reasons. see below.)
 - **s3_bucket**: S3 bucket name for temporary files
 - **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
 - **options**: extra connection properties (hash, default: {})

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -22,6 +22,7 @@ import org.embulk.output.postgresql.AbstractPostgreSQLCopyBatchInsert;
 import org.embulk.spi.Exec;
 import org.slf4j.Logger;
 
+import com.amazonaws.AmazonClientException;
 import com.amazonaws.auth.AWSCredentialsProvider;
 import com.amazonaws.auth.BasicSessionCredentials;
 import com.amazonaws.auth.policy.Policy;
@@ -30,6 +31,7 @@ import com.amazonaws.auth.policy.Statement;
 import com.amazonaws.auth.policy.Statement.Effect;
 import com.amazonaws.auth.policy.actions.S3Actions;
 import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.model.Region;
 import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
 import com.amazonaws.services.securitytoken.model.Credentials;
 import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
@@ -45,6 +47,7 @@ public class RedshiftCopyBatchInsert
     private final String iamReaderUserName;
     private final AWSCredentialsProvider credentialsProvider;
     private final AmazonS3Client s3;
+    private final String s3RegionName;
     private final AWSSecurityTokenServiceClient sts;
     private final ExecutorService executorService;
 
@@ -72,9 +75,21 @@ public class RedshiftCopyBatchInsert
         this.credentialsProvider = credentialsProvider;
         this.s3 = new AmazonS3Client(credentialsProvider);  // TODO options
         this.sts = new AWSSecurityTokenServiceClient(credentialsProvider);  // options
-
         this.executorService = Executors.newCachedThreadPool();
         this.uploadAndCopyFutures = new ArrayList<Future<Void>>();
+
+        String s3RegionName = null;
+        try {
+            String s3Location = s3.getBucketLocation(s3BucketName);
+            Region s3Region = Region.fromValue(s3Location);
+            com.amazonaws.regions.Region region = s3Region.toAWSRegion();
+            s3RegionName = region.getName();
+            logger.info("S3 region for bucket '" + s3BucketName + "' is '" + s3RegionName + "'.");
+        } catch (AmazonClientException | IllegalArgumentException e) {
+            logger.warn("Cannot get S3 region for bucket '" + s3BucketName + "'."
+                    + " IAM user needs \"s3:GetBucketLocation\" permission if Redshift region and S3 region are different.");
+        }
+        this.s3RegionName = s3RegionName;
     }
 
     @Override
@@ -274,6 +289,12 @@ public class RedshiftCopyBatchInsert
                 sb.append(creds.getSessionToken());
             }
             sb.append("' ");
+            if (s3RegionName != null) {
+                sb.append("REGION '");
+                sb.append(s3RegionName);
+                sb.append("' ");
+            }
+
             sb.append(COPY_AFTER_FROM);
             return sb.toString();
         }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.6.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-18 00:00:00.000000000 Z
+date: 2016-09-30 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -19,24 +19,24 @@ extra_rdoc_files: []
 files:
 - README.md
 - build.gradle
-- lib/embulk/output/redshift.rb
-- src/main/java/org/embulk/output/RedshiftOutputPlugin.java
-- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
-- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
-- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
-- src/main/java/org/embulk/output/redshift/Ssl.java
 - classpath/aws-java-sdk-core-1.10.33.jar
 - classpath/aws-java-sdk-kms-1.10.33.jar
 - classpath/aws-java-sdk-s3-1.10.33.jar
 - classpath/aws-java-sdk-sts-1.10.33.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-jdbc-0.6.3.jar
-- classpath/embulk-output-postgresql-0.6.3.jar
-- classpath/embulk-output-redshift-0.6.3.jar
+- classpath/embulk-output-jdbc-0.6.4.jar
+- classpath/embulk-output-postgresql-0.6.4.jar
+- classpath/embulk-output-redshift-0.6.4.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
+- lib/embulk/output/redshift.rb
+- src/main/java/org/embulk/output/RedshiftOutputPlugin.java
+- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
+- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
+- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
+- src/main/java/org/embulk/output/redshift/Ssl.java
 homepage: https://github.com/embulk/embulk-output-jdbc
 licenses:
 - Apache 2.0
@@ -47,17 +47,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: JDBC output plugin for Embulk