embulk-output-redshift 0.6.3 → 0.6.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/classpath/{embulk-output-jdbc-0.6.3.jar → embulk-output-jdbc-0.6.4.jar} +0 -0
- data/classpath/{embulk-output-postgresql-0.6.3.jar → embulk-output-postgresql-0.6.4.jar} +0 -0
- data/classpath/embulk-output-redshift-0.6.4.jar +0 -0
- data/src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java +22 -1
- metadata +14 -14
- data/classpath/embulk-output-redshift-0.6.3.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4f9b1303b4c1343a10b77b15b07fca1536c919ac
|
4
|
+
data.tar.gz: 3fe7932bc6a6a3d0125c88e9e60c8e9ac05cb4d6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e06bf310d402011178bb190ae8ad2240ad97ddebba96e54f681e49890e52de6cb5175c6a78152bb124a70159a345a21ad5aa67745e145a34116598736f4e28bf
|
7
|
+
data.tar.gz: 434386a4927d5f7df605142e93c5870ec3c7b3194bac5c7d3b0b56161c51beec09f44480510dcbe290412a3846c125874caa62b95a9651ac1631738a457eaad0
|
data/README.md
CHANGED
@@ -20,7 +20,7 @@ Redshift output plugins for Embulk loads records to Redshift.
|
|
20
20
|
- **table**: destination table name (string, required)
|
21
21
|
- **access_key_id**: access key id for AWS
|
22
22
|
- **secret_access_key**: secret access key for AWS
|
23
|
-
- **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`,
|
23
|
+
- **iam_user_name**: IAM user name for uploading temporary files to S3. The user should have permissions of `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:ListBucket` and `sts:GetFederationToken`. And furthermore, the user should have permission of `s3:GetBucketLocation` if Redshift region and S3 bucket region are different. (string, default: "", but we strongly recommend that you use IAM user for security reasons. see below.)
|
24
24
|
- **s3_bucket**: S3 bucket name for temporary files
|
25
25
|
- **s3_key_prefix**: S3 key prefix for temporary files (string, default:"")
|
26
26
|
- **options**: extra connection properties (hash, default: {})
|
Binary file
|
Binary file
|
Binary file
|
@@ -22,6 +22,7 @@ import org.embulk.output.postgresql.AbstractPostgreSQLCopyBatchInsert;
|
|
22
22
|
import org.embulk.spi.Exec;
|
23
23
|
import org.slf4j.Logger;
|
24
24
|
|
25
|
+
import com.amazonaws.AmazonClientException;
|
25
26
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
26
27
|
import com.amazonaws.auth.BasicSessionCredentials;
|
27
28
|
import com.amazonaws.auth.policy.Policy;
|
@@ -30,6 +31,7 @@ import com.amazonaws.auth.policy.Statement;
|
|
30
31
|
import com.amazonaws.auth.policy.Statement.Effect;
|
31
32
|
import com.amazonaws.auth.policy.actions.S3Actions;
|
32
33
|
import com.amazonaws.services.s3.AmazonS3Client;
|
34
|
+
import com.amazonaws.services.s3.model.Region;
|
33
35
|
import com.amazonaws.services.securitytoken.AWSSecurityTokenServiceClient;
|
34
36
|
import com.amazonaws.services.securitytoken.model.Credentials;
|
35
37
|
import com.amazonaws.services.securitytoken.model.GetFederationTokenRequest;
|
@@ -45,6 +47,7 @@ public class RedshiftCopyBatchInsert
|
|
45
47
|
private final String iamReaderUserName;
|
46
48
|
private final AWSCredentialsProvider credentialsProvider;
|
47
49
|
private final AmazonS3Client s3;
|
50
|
+
private final String s3RegionName;
|
48
51
|
private final AWSSecurityTokenServiceClient sts;
|
49
52
|
private final ExecutorService executorService;
|
50
53
|
|
@@ -72,9 +75,21 @@ public class RedshiftCopyBatchInsert
|
|
72
75
|
this.credentialsProvider = credentialsProvider;
|
73
76
|
this.s3 = new AmazonS3Client(credentialsProvider); // TODO options
|
74
77
|
this.sts = new AWSSecurityTokenServiceClient(credentialsProvider); // options
|
75
|
-
|
76
78
|
this.executorService = Executors.newCachedThreadPool();
|
77
79
|
this.uploadAndCopyFutures = new ArrayList<Future<Void>>();
|
80
|
+
|
81
|
+
String s3RegionName = null;
|
82
|
+
try {
|
83
|
+
String s3Location = s3.getBucketLocation(s3BucketName);
|
84
|
+
Region s3Region = Region.fromValue(s3Location);
|
85
|
+
com.amazonaws.regions.Region region = s3Region.toAWSRegion();
|
86
|
+
s3RegionName = region.getName();
|
87
|
+
logger.info("S3 region for bucket '" + s3BucketName + "' is '" + s3RegionName + "'.");
|
88
|
+
} catch (AmazonClientException | IllegalArgumentException e) {
|
89
|
+
logger.warn("Cannot get S3 region for bucket '" + s3BucketName + "'."
|
90
|
+
+ " IAM user needs \"s3:GetBucketLocation\" permission if Redshift region and S3 region are different.");
|
91
|
+
}
|
92
|
+
this.s3RegionName = s3RegionName;
|
78
93
|
}
|
79
94
|
|
80
95
|
@Override
|
@@ -274,6 +289,12 @@ public class RedshiftCopyBatchInsert
|
|
274
289
|
sb.append(creds.getSessionToken());
|
275
290
|
}
|
276
291
|
sb.append("' ");
|
292
|
+
if (s3RegionName != null) {
|
293
|
+
sb.append("REGION '");
|
294
|
+
sb.append(s3RegionName);
|
295
|
+
sb.append("' ");
|
296
|
+
}
|
297
|
+
|
277
298
|
sb.append(COPY_AFTER_FROM);
|
278
299
|
return sb.toString();
|
279
300
|
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-redshift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-09-30 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Inserts or updates records to a table.
|
14
14
|
email:
|
@@ -19,24 +19,24 @@ extra_rdoc_files: []
|
|
19
19
|
files:
|
20
20
|
- README.md
|
21
21
|
- build.gradle
|
22
|
-
- lib/embulk/output/redshift.rb
|
23
|
-
- src/main/java/org/embulk/output/RedshiftOutputPlugin.java
|
24
|
-
- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
|
25
|
-
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
|
26
|
-
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
|
27
|
-
- src/main/java/org/embulk/output/redshift/Ssl.java
|
28
22
|
- classpath/aws-java-sdk-core-1.10.33.jar
|
29
23
|
- classpath/aws-java-sdk-kms-1.10.33.jar
|
30
24
|
- classpath/aws-java-sdk-s3-1.10.33.jar
|
31
25
|
- classpath/aws-java-sdk-sts-1.10.33.jar
|
32
26
|
- classpath/commons-codec-1.6.jar
|
33
27
|
- classpath/commons-logging-1.1.3.jar
|
34
|
-
- classpath/embulk-output-jdbc-0.6.
|
35
|
-
- classpath/embulk-output-postgresql-0.6.
|
36
|
-
- classpath/embulk-output-redshift-0.6.
|
28
|
+
- classpath/embulk-output-jdbc-0.6.4.jar
|
29
|
+
- classpath/embulk-output-postgresql-0.6.4.jar
|
30
|
+
- classpath/embulk-output-redshift-0.6.4.jar
|
37
31
|
- classpath/httpclient-4.3.6.jar
|
38
32
|
- classpath/httpcore-4.3.3.jar
|
39
33
|
- classpath/postgresql-9.4-1205-jdbc41.jar
|
34
|
+
- lib/embulk/output/redshift.rb
|
35
|
+
- src/main/java/org/embulk/output/RedshiftOutputPlugin.java
|
36
|
+
- src/main/java/org/embulk/output/redshift/RedshiftCopyBatchInsert.java
|
37
|
+
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnection.java
|
38
|
+
- src/main/java/org/embulk/output/redshift/RedshiftOutputConnector.java
|
39
|
+
- src/main/java/org/embulk/output/redshift/Ssl.java
|
40
40
|
homepage: https://github.com/embulk/embulk-output-jdbc
|
41
41
|
licenses:
|
42
42
|
- Apache 2.0
|
@@ -47,17 +47,17 @@ require_paths:
|
|
47
47
|
- lib
|
48
48
|
required_ruby_version: !ruby/object:Gem::Requirement
|
49
49
|
requirements:
|
50
|
-
- -
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
52
|
version: '0'
|
53
53
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
54
54
|
requirements:
|
55
|
-
- -
|
55
|
+
- - ">="
|
56
56
|
- !ruby/object:Gem::Version
|
57
57
|
version: '0'
|
58
58
|
requirements: []
|
59
59
|
rubyforge_project:
|
60
|
-
rubygems_version: 2.
|
60
|
+
rubygems_version: 2.4.8
|
61
61
|
signing_key:
|
62
62
|
specification_version: 4
|
63
63
|
summary: JDBC output plugin for Embulk
|
Binary file
|