embulk-input-s3 0.2.8 → 0.2.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/build.gradle +4 -0
- data/classpath/embulk-input-s3-0.2.9.jar +0 -0
- data/classpath/embulk-util-aws-credentials-0.2.9.jar +0 -0
- data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +2 -0
- metadata +4 -5
- data/classpath/embulk-input-s3-0.2.8.jar +0 -0
- data/src/main/java/org/embulk/input/s3/AwsCredentials.java +0 -179
- data/src/main/java/org/embulk/input/s3/AwsCredentialsTask.java +0 -39
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 581dfa840436ba5dddad8e98bf48a464da5fc408
|
4
|
+
data.tar.gz: 7451197f321556c6da2e2ff7ecd4575add015595
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ac5f4ce10e60ec00b050351e08ecda5cd5362a8282dc71a81268ee10bb6c93aae05d73b6e8489aab1ef55fdc43a7e763307e39bf0edecf0704753e0eb3124226
|
7
|
+
data.tar.gz: d7b7dc34bb59cb870ec31152db8a9178e44743ad5511ce95dda4cf00e368057d66b9ee3de4dfa2421f19b3431959c79fb2eecad5349288c864b5ac155b4f215d
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -41,6 +41,8 @@ import org.embulk.spi.util.InputStreamFileInput;
|
|
41
41
|
import org.embulk.spi.util.ResumableInputStream;
|
42
42
|
import org.embulk.spi.util.RetryExecutor.Retryable;
|
43
43
|
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
44
|
+
import org.embulk.util.aws.credentials.AwsCredentials;
|
45
|
+
import org.embulk.util.aws.credentials.AwsCredentialsTask;
|
44
46
|
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
45
47
|
|
46
48
|
public abstract class AbstractS3FileInputPlugin
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-03-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -48,8 +48,6 @@ files:
|
|
48
48
|
- build.gradle
|
49
49
|
- lib/embulk/input/s3.rb
|
50
50
|
- src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
|
51
|
-
- src/main/java/org/embulk/input/s3/AwsCredentials.java
|
52
|
-
- src/main/java/org/embulk/input/s3/AwsCredentialsTask.java
|
53
51
|
- src/main/java/org/embulk/input/s3/FileList.java
|
54
52
|
- src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
|
55
53
|
- src/test/java/org/embulk/input/s3/TestAwsCredentials.java
|
@@ -61,7 +59,8 @@ files:
|
|
61
59
|
- classpath/aws-java-sdk-kms-1.10.33.jar
|
62
60
|
- classpath/aws-java-sdk-s3-1.10.33.jar
|
63
61
|
- classpath/commons-codec-1.6.jar
|
64
|
-
- classpath/embulk-input-s3-0.2.
|
62
|
+
- classpath/embulk-input-s3-0.2.9.jar
|
63
|
+
- classpath/embulk-util-aws-credentials-0.2.9.jar
|
65
64
|
- classpath/httpclient-4.3.6.jar
|
66
65
|
- classpath/httpcore-4.3.3.jar
|
67
66
|
- classpath/jcl-over-slf4j-1.7.12.jar
|
Binary file
|
@@ -1,179 +0,0 @@
|
|
1
|
-
package org.embulk.input.s3;
|
2
|
-
|
3
|
-
import com.google.common.base.Optional;
|
4
|
-
import com.amazonaws.auth.AWSCredentials;
|
5
|
-
import com.amazonaws.auth.AWSCredentialsProvider;
|
6
|
-
import com.amazonaws.auth.AWSSessionCredentials;
|
7
|
-
import com.amazonaws.auth.AWSSessionCredentialsProvider;
|
8
|
-
import com.amazonaws.auth.AnonymousAWSCredentials;
|
9
|
-
import com.amazonaws.auth.BasicAWSCredentials;
|
10
|
-
import com.amazonaws.auth.BasicSessionCredentials;
|
11
|
-
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
|
12
|
-
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
13
|
-
import com.amazonaws.auth.SystemPropertiesCredentialsProvider;
|
14
|
-
import com.amazonaws.auth.profile.ProfileCredentialsProvider;
|
15
|
-
import com.amazonaws.auth.profile.ProfilesConfigFile;
|
16
|
-
import org.embulk.config.ConfigException;
|
17
|
-
import org.embulk.spi.Exec;
|
18
|
-
import org.embulk.spi.unit.LocalFile;
|
19
|
-
import org.slf4j.Logger;
|
20
|
-
|
21
|
-
public abstract class AwsCredentials
|
22
|
-
{
|
23
|
-
private AwsCredentials() { }
|
24
|
-
|
25
|
-
public static AWSCredentialsProvider getAWSCredentialsProvider(AwsCredentialsTask task)
|
26
|
-
{
|
27
|
-
switch (task.getAuthMethod()) {
|
28
|
-
case "basic":
|
29
|
-
// for backward compatibility
|
30
|
-
if (!task.getAccessKeyId().isPresent() && !task.getAccessKeyId().isPresent()) {
|
31
|
-
final Logger log = Exec.getLogger(AwsCredentials.class);
|
32
|
-
log.warn("Both access_key_id and secret_access_key are not set. Assuming that 'auth_method: anonymous' option is set.");
|
33
|
-
log.warn("If you intentionally use anonymous authentication, please set 'auth_method: anonymous' option.");
|
34
|
-
log.warn("This behavior will be removed in a futurte release.");
|
35
|
-
reject(task.getSessionToken(), "session_token");
|
36
|
-
reject(task.getProfileFile(), "profile_file");
|
37
|
-
reject(task.getProfileName(), "profile_name");
|
38
|
-
return new AWSCredentialsProvider() {
|
39
|
-
public AWSCredentials getCredentials()
|
40
|
-
{
|
41
|
-
return new AnonymousAWSCredentials();
|
42
|
-
}
|
43
|
-
|
44
|
-
public void refresh() { }
|
45
|
-
};
|
46
|
-
}
|
47
|
-
else {
|
48
|
-
String accessKeyId = require(task.getAccessKeyId(), "'access_key_id', 'secret_access_key'");
|
49
|
-
String secretAccessKey = require(task.getSecretAccessKey(), "'secret_access_key'");
|
50
|
-
reject(task.getSessionToken(), "session_token");
|
51
|
-
reject(task.getProfileFile(), "profile_file");
|
52
|
-
reject(task.getProfileName(), "profile_name");
|
53
|
-
final BasicAWSCredentials creds = new BasicAWSCredentials(accessKeyId, secretAccessKey);
|
54
|
-
return new AWSCredentialsProvider() {
|
55
|
-
public AWSCredentials getCredentials()
|
56
|
-
{
|
57
|
-
return creds;
|
58
|
-
}
|
59
|
-
|
60
|
-
public void refresh() { }
|
61
|
-
};
|
62
|
-
}
|
63
|
-
|
64
|
-
case "env":
|
65
|
-
reject(task.getAccessKeyId(), "access_key_id");
|
66
|
-
reject(task.getSecretAccessKey(), "secret_access_key");
|
67
|
-
reject(task.getSessionToken(), "session_token");
|
68
|
-
reject(task.getProfileFile(), "profile_file");
|
69
|
-
reject(task.getProfileName(), "profile_name");
|
70
|
-
return overwriteBasicCredentials(task, new EnvironmentVariableCredentialsProvider().getCredentials());
|
71
|
-
|
72
|
-
case "instance":
|
73
|
-
reject(task.getAccessKeyId(), "access_key_id");
|
74
|
-
reject(task.getSecretAccessKey(), "secret_access_key");
|
75
|
-
reject(task.getSessionToken(), "session_token");
|
76
|
-
reject(task.getProfileFile(), "profile_file");
|
77
|
-
reject(task.getProfileName(), "profile_name");
|
78
|
-
return new InstanceProfileCredentialsProvider();
|
79
|
-
|
80
|
-
case "profile":
|
81
|
-
{
|
82
|
-
reject(task.getAccessKeyId(), "access_key_id");
|
83
|
-
reject(task.getSecretAccessKey(), "secret_access_key");
|
84
|
-
reject(task.getSessionToken(), "session_token");
|
85
|
-
|
86
|
-
String profileName = task.getProfileName().or("default");
|
87
|
-
ProfileCredentialsProvider provider;
|
88
|
-
if (task.getProfileFile().isPresent()) {
|
89
|
-
ProfilesConfigFile file = new ProfilesConfigFile(task.getProfileFile().get().getFile());
|
90
|
-
provider = new ProfileCredentialsProvider(file, profileName);
|
91
|
-
}
|
92
|
-
else {
|
93
|
-
provider = new ProfileCredentialsProvider(profileName);
|
94
|
-
}
|
95
|
-
task.setProfileName(Optional.<String>absent());
|
96
|
-
task.setProfileFile(Optional.<LocalFile>absent());
|
97
|
-
|
98
|
-
return overwriteBasicCredentials(task, provider.getCredentials());
|
99
|
-
}
|
100
|
-
|
101
|
-
case "properties":
|
102
|
-
reject(task.getAccessKeyId(), "access_key_id");
|
103
|
-
reject(task.getSecretAccessKey(), "secret_access_key");
|
104
|
-
reject(task.getSessionToken(), "session_token");
|
105
|
-
reject(task.getProfileFile(), "profile_file");
|
106
|
-
reject(task.getProfileName(), "profile_name");
|
107
|
-
return overwriteBasicCredentials(task, new SystemPropertiesCredentialsProvider().getCredentials());
|
108
|
-
|
109
|
-
case "anonymous":
|
110
|
-
reject(task.getAccessKeyId(), "access_key_id");
|
111
|
-
reject(task.getSecretAccessKey(), "secret_access_key");
|
112
|
-
reject(task.getSessionToken(), "session_token");
|
113
|
-
reject(task.getProfileFile(), "profile_file");
|
114
|
-
reject(task.getProfileName(), "profile_name");
|
115
|
-
return new AWSCredentialsProvider() {
|
116
|
-
public AWSCredentials getCredentials()
|
117
|
-
{
|
118
|
-
return new AnonymousAWSCredentials();
|
119
|
-
}
|
120
|
-
|
121
|
-
public void refresh() { }
|
122
|
-
};
|
123
|
-
|
124
|
-
case "session":
|
125
|
-
{
|
126
|
-
String accessKeyId = require(task.getAccessKeyId(), "'access_key_id', 'secret_access_key', 'session_token'");
|
127
|
-
String secretAccessKey = require(task.getSecretAccessKey(), "'secret_access_key', 'session_token'");
|
128
|
-
String sessionToken = require(task.getSessionToken(), "'session_token'");
|
129
|
-
reject(task.getProfileFile(), "profile_file");
|
130
|
-
reject(task.getProfileName(), "profile_name");
|
131
|
-
final AWSSessionCredentials creds = new BasicSessionCredentials(accessKeyId, secretAccessKey, sessionToken);
|
132
|
-
return new AWSSessionCredentialsProvider() {
|
133
|
-
public AWSSessionCredentials getCredentials()
|
134
|
-
{
|
135
|
-
return creds;
|
136
|
-
}
|
137
|
-
|
138
|
-
public void refresh() { }
|
139
|
-
};
|
140
|
-
}
|
141
|
-
|
142
|
-
default:
|
143
|
-
throw new ConfigException(String.format("Unknwon auth_method '%s'. Supported methods are basic, instance, profile, properties, anonymous, and session.",
|
144
|
-
task.getAuthMethod()));
|
145
|
-
}
|
146
|
-
}
|
147
|
-
|
148
|
-
private static AWSCredentialsProvider overwriteBasicCredentials(AwsCredentialsTask task, final AWSCredentials creds)
|
149
|
-
{
|
150
|
-
task.setAuthMethod("basic");
|
151
|
-
task.setAccessKeyId(Optional.of(creds.getAWSAccessKeyId()));
|
152
|
-
task.setSecretAccessKey(Optional.of(creds.getAWSSecretKey()));
|
153
|
-
return new AWSCredentialsProvider() {
|
154
|
-
public AWSCredentials getCredentials()
|
155
|
-
{
|
156
|
-
return creds;
|
157
|
-
}
|
158
|
-
|
159
|
-
public void refresh() { }
|
160
|
-
};
|
161
|
-
}
|
162
|
-
|
163
|
-
private static <T> T require(Optional<T> value, String message)
|
164
|
-
{
|
165
|
-
if (value.isPresent()) {
|
166
|
-
return value.get();
|
167
|
-
}
|
168
|
-
else {
|
169
|
-
throw new ConfigException("Required option is not set: " + message);
|
170
|
-
}
|
171
|
-
}
|
172
|
-
|
173
|
-
private static <T> void reject(Optional<T> value, String message)
|
174
|
-
{
|
175
|
-
if (value.isPresent()) {
|
176
|
-
throw new ConfigException("Invalid option is set: " + message);
|
177
|
-
}
|
178
|
-
}
|
179
|
-
}
|
@@ -1,39 +0,0 @@
|
|
1
|
-
package org.embulk.input.s3;
|
2
|
-
|
3
|
-
import com.google.common.base.Optional;
|
4
|
-
import org.embulk.config.Config;
|
5
|
-
import org.embulk.config.ConfigDefault;
|
6
|
-
import org.embulk.spi.unit.LocalFile;
|
7
|
-
|
8
|
-
public interface AwsCredentialsTask
|
9
|
-
{
|
10
|
-
@Config("auth_method")
|
11
|
-
@ConfigDefault("\"basic\"")
|
12
|
-
String getAuthMethod();
|
13
|
-
void setAuthMethod(String method);
|
14
|
-
|
15
|
-
@Config("access_key_id")
|
16
|
-
@ConfigDefault("null")
|
17
|
-
Optional<String> getAccessKeyId();
|
18
|
-
void setAccessKeyId(Optional<String> value);
|
19
|
-
|
20
|
-
@Config("secret_access_key")
|
21
|
-
@ConfigDefault("null")
|
22
|
-
Optional<String> getSecretAccessKey();
|
23
|
-
void setSecretAccessKey(Optional<String> value);
|
24
|
-
|
25
|
-
@Config("session_token")
|
26
|
-
@ConfigDefault("null")
|
27
|
-
Optional<String> getSessionToken();
|
28
|
-
void setSessionToken(Optional<String> value);
|
29
|
-
|
30
|
-
@Config("profile_file")
|
31
|
-
@ConfigDefault("null")
|
32
|
-
Optional<LocalFile> getProfileFile();
|
33
|
-
void setProfileFile(Optional<LocalFile> value);
|
34
|
-
|
35
|
-
@Config("profile_name")
|
36
|
-
@ConfigDefault("null")
|
37
|
-
Optional<String> getProfileName();
|
38
|
-
void setProfileName(Optional<String> value);
|
39
|
-
}
|