embulk-output-redshift 0.7.1 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47ca8a15eaed456d7a006c1dfdb43b2ae85b9013
-  data.tar.gz: 1fa61b84e89e32e81c7fb748d8d2edcbb4bf4481
+  metadata.gz: f328186471e1367510fec732ad9747551820ac5a
+  data.tar.gz: 7627fe90dd396daf16b047f85f5d2084a5a91944
 SHA512:
-  metadata.gz: b9652025ca46d67558454940362e804477d7cfa1a93bd5f19accc8a42b6bbdb11f2930bf514398b51c3bd68d9053e6258aa80badff2dd0b01329af25de6d1f12
-  data.tar.gz: 503904556c128353ad4ef366f7b0ed31300a2f9ea5c93eb0688c418a7c08a266d2ed48e65e6a93dce4fcb3f78cfb78e29e196b2241563a949a8863100a70bb34
+  metadata.gz: 5c8671c622847596ce3b703b32a6e4c0a1a5d28c8f6de3c941293cbcd9a4f2faba199f2a320cc3f6110b86b9651d0c6524d5eae5c7f03accc49db3d4ed89360f
+  data.tar.gz: bb2d580607179e5ab0cdc402af4fc1e1ba11d463a905bf11bf61253de1d2d224e79029fda08fc0609773a3dfc3eaceb3b2a8f000f788bea2cd4c3f7fcae82499

data/README.md

@@ -18,8 +18,47 @@ Redshift output plugin for Embulk loads records to Redshift.
 - **database**: destination database name (string, required)
 - **schema**: destination schema name (string, default: "public")
 - **table**: destination table name (string, required)
-- **access_key_id**: access key id for AWS
-- **secret_access_key**: secret access key for AWS
+- **access_key_id**: deprecated. `aws_access_key_id` should be used (see "basic" in `aws_auth_method`).
+- **secret_access_key**: deprecated. `aws_secret_access_key` should be used (see "basic" in `aws_auth_method`).
+- **aws_auth_method**: name of mechanism to authenticate requests ("basic", "env", "instance", "profile", "properties", "anonymous", or "session". default: "basic")
+
+  - "basic": uses `access_key_id` and `secret_access_key` to authenticate.
+
+    - **aws_access_key_id**: AWS access key ID (string, required)
+
+    - **aws_secret_access_key**: AWS secret access key (string, required)
+
+  - "env": uses `AWS_ACCESS_KEY_ID` (or `AWS_ACCESS_KEY`) and `AWS_SECRET_KEY` (or `AWS_SECRET_ACCESS_KEY`) environment variables.
+
+  - "instance": uses EC2 instance profile.
+
+  - "profile": uses credentials written in a file. Format of the file is as following, where `[...]` is a name of profile.
+
+    - **aws_profile_file**: path to a profiles file. (string, default: given by `AWS_CREDENTIAL_PROFILES_FILE` environment varialbe, or ~/.aws/credentials).
+
+    - **aws_profile_name**: name of a profile. (string, default: `"default"`)
+
+    ```
+    [default]
+    aws_access_key_id=YOUR_ACCESS_KEY_ID
+    aws_secret_access_key=YOUR_SECRET_ACCESS_KEY
+
+    [profile2]
+    ...
+    ```
+
+  - "properties": uses `aws.accessKeyId` and `aws.secretKey` Java system properties.
+
+  - "anonymous": uses anonymous access. This authentication method can access only public files.
+
+  - "session": uses temporary-generated `access_key_id`, `secret_access_key` and `session_token`.
+
+    - **aws_access_key_id**: AWS access key ID (string, required)
+
+    - **aws_secret_access_key**: AWS secret access key (string, required)
+
+    - **aws_session_token**: session token (string, required)
+
 - **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` and `sts:GetFederationToken`. And furthermore, the user should have permission of `s3:GetBucketLocation` if Redshift region and S3 bucket region are different. (string, default: "", but we strongly recommend that you use IAM user for security reasons. see below.)
 - **s3_bucket**: S3 bucket name for temporary files
 - **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
@@ -89,8 +128,8 @@ out:
   password: ""
   database: my_database
   table: my_table
-  access_key_id: ABCXYZ123ABCXYZ123
-  secret_access_key: AbCxYz123aBcXyZ123
+  aws_access_key_id: ABCXYZ123ABCXYZ123
+  aws_secret_access_key: AbCxYz123aBcXyZ123
   iam_user_name: my-s3-read-only
   s3_bucket: my-redshift-transfer-bucket
   s3_key_prefix: temp/redshift
@@ -108,8 +147,8 @@ out:
   password: ""
   database: my_database
   table: my_table
-  access_key_id: ABCXYZ123ABCXYZ123
-  secret_access_key: AbCxYz123aBcXyZ123
+  aws_access_key_id: ABCXYZ123ABCXYZ123
+  aws_secret_access_key: AbCxYz123aBcXyZ123
   iam_user_name: my-s3-read-only
   s3_bucket: my-redshift-transfer-bucket
   s3_key_prefix: temp/redshift
@@ -122,6 +161,38 @@ out:
     my_col_5: {type: 'DECIMAL(18,9)', value_type: pass}
 ```
 
+To use IAM Role:
+
+```yaml
+out:
+  type: redshift
+  host: myinstance.us-west-2.redshift.amazonaws.com
+  user: pg
+  password: ""
+  database: my_database
+  table: my_table
+  s3_bucket: my-redshift-transfer-bucket
+  s3_key_prefix: temp/redshift
+  mode: insert
+  aws_auth_method: instance
+```
+
+To use AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables:
+
+```yaml
+out:
+  type: redshift
+  host: myinstance.us-west-2.redshift.amazonaws.com
+  user: pg
+  password: ""
+  database: my_database
+  table: my_table
+  s3_bucket: my-redshift-transfer-bucket
+  s3_key_prefix: temp/redshift
+  mode: insert
+  aws_auth_method: env
+```
+
 ### Build
 
 ```

data/build.gradle

@@ -4,6 +4,7 @@ dependencies {
 
     compile "com.amazonaws:aws-java-sdk-s3:1.10.33"
     compile "com.amazonaws:aws-java-sdk-sts:1.10.33"
+	compile 'org.embulk.input.s3:embulk-util-aws-credentials:0.2.8'
 
     testCompile project(':embulk-output-jdbc').sourceSets.test.output
 }

data/src/main/java/org/embulk/output/RedshiftOutputPlugin.java

@@ -1,6 +1,5 @@
 package org.embulk.output;
 
-import java.util.List;
 import java.util.Properties;
 import java.io.IOException;
 import java.sql.SQLException;
@@ -9,10 +8,10 @@ import org.embulk.output.jdbc.MergeConfig;
 import org.slf4j.Logger;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableSet;
-import com.amazonaws.auth.AWSCredentials;
 import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.auth.BasicAWSCredentials;
 import org.embulk.spi.Exec;
+import org.embulk.util.aws.credentials.AwsCredentials;
+import org.embulk.util.aws.credentials.AwsCredentialsTaskWithPrefix;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.output.jdbc.AbstractJdbcOutputPlugin;
@@ -26,7 +25,7 @@ public class RedshiftOutputPlugin
 {
     private final Logger logger = Exec.getLogger(RedshiftOutputPlugin.class);
 
-    public interface RedshiftPluginTask extends PluginTask
+    public interface RedshiftPluginTask extends AwsCredentialsTaskWithPrefix, PluginTask
     {
         @Config("host")
         public String getHost();
@@ -49,11 +48,15 @@ public class RedshiftOutputPlugin
         @ConfigDefault("\"public\"")
         public String getSchema();
 
+        // for backward compatibility
         @Config("access_key_id")
-        public String getAccessKeyId();
+        @ConfigDefault("null")
+        Optional<String> getOldAccessKeyId();
 
+        // for backward compatibility
         @Config("secret_access_key")
-        public String getSecretAccessKey();
+        @ConfigDefault("null")
+        Optional<String> getOldSecretAccessKey();
 
         @Config("iam_user_name")
         @ConfigDefault("\"\"")
@@ -132,20 +135,21 @@ public class RedshiftOutputPlugin
 
     private static AWSCredentialsProvider getAWSCredentialsProvider(RedshiftPluginTask task)
     {
-        final AWSCredentials creds = new BasicAWSCredentials(
-                task.getAccessKeyId(), task.getSecretAccessKey());
-        return new AWSCredentialsProvider() {
-            @Override
-            public AWSCredentials getCredentials()
-            {
-                return creds;
-            }
+        return AwsCredentials.getAWSCredentialsProvider(task);
+    }
 
-            @Override
-            public void refresh()
-            {
+    private void setAWSCredentialsBackwardCompatibility(RedshiftPluginTask t)
+    {
+        if ("basic".equals(t.getAuthMethod())) {
+            if (t.getOldAccessKeyId().isPresent() && !t.getAccessKeyId().isPresent()) {
+                logger.warn("'access_key_id' is deprecated. Please use 'aws_access_key_id'.");
+                t.setAccessKeyId(t.getOldAccessKeyId());
             }
-        };
+            if (t.getOldSecretAccessKey().isPresent() && !t.getSecretAccessKey().isPresent()) {
+                logger.warn("'secret_access_key' is deprecated. Please use 'aws_secret_access_key'.");
+                t.setSecretAccessKey(t.getOldSecretAccessKey());
+            }
+        }
     }
 
     @Override
@@ -155,6 +159,7 @@ public class RedshiftOutputPlugin
             throw new UnsupportedOperationException("Redshift output plugin doesn't support 'merge_direct' mode. Use 'merge' mode instead.");
         }
         RedshiftPluginTask t = (RedshiftPluginTask) task;
+        setAWSCredentialsBackwardCompatibility(t);
         return new RedshiftCopyBatchInsert(getConnector(task, true),
                 getAWSCredentialsProvider(t), t.getS3Bucket(), t.getS3KeyPrefix(), t.getIamUserName());
     }

data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java

@@ -194,6 +194,10 @@ public class RedshiftCopyBatchInsert
                     c.getSecretAccessKey(),
                     c.getSessionToken());
         } else {
+            if (credentialsProvider.getCredentials() instanceof BasicSessionCredentials) {
+                BasicSessionCredentials credentials = (BasicSessionCredentials) credentialsProvider.getCredentials();
+                return credentials;
+            }
             return new BasicSessionCredentials(credentialsProvider.getCredentials().getAWSAccessKeyId(),
                     credentialsProvider.getCredentials().getAWSSecretKey(), null);
         }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-redshift
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.7.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-25 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies: []
 description: Inserts or updates records to a table.
 email:
@@ -25,11 +25,15 @@ files:
 - classpath/aws-java-sdk-sts-1.10.33.jar
 - classpath/commons-codec-1.6.jar
 - classpath/commons-logging-1.1.3.jar
-- classpath/embulk-output-jdbc-0.7.1.jar
-- classpath/embulk-output-postgresql-0.7.1.jar
-- classpath/embulk-output-redshift-0.7.1.jar
+- classpath/embulk-core-0.8.9.jar
+- classpath/embulk-output-jdbc-0.7.2.jar
+- classpath/embulk-output-postgresql-0.7.2.jar
+- classpath/embulk-output-redshift-0.7.2.jar
+- classpath/embulk-util-aws-credentials-0.2.8.jar
 - classpath/httpclient-4.3.6.jar
 - classpath/httpcore-4.3.3.jar
+- classpath/jcl-over-slf4j-1.7.12.jar
+- classpath/msgpack-core-0.8.7.jar
 - classpath/postgresql-9.4-1205-jdbc41.jar
 - lib/embulk/output/redshift.rb
 - src/main/java/org/embulk/output/RedshiftOutputPlugin.java