embulk-input-s3 0.2.10 → 0.2.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-input-s3-0.2.11.jar +0 -0
- data/classpath/{embulk-util-aws-credentials-0.2.10.jar → embulk-util-aws-credentials-0.2.11.jar} +0 -0
- data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +37 -0
- data/src/main/java/org/embulk/input/s3/HttpProxy.java +35 -0
- data/src/test/java/org/embulk/input/s3/TestHttpProxy.java +120 -0
- metadata +6 -4
- data/classpath/embulk-input-s3-0.2.10.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 034a5456271657abde4e9dc5aac76f7e05efc7e4
|
4
|
+
data.tar.gz: 74f38ae4046a4292487361a81d74c3be201b5aaa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3cc77fc7cf409ce21e2a7373fc983727953a0d22b3c5676c43f4a4c3182a15de487ec3633d3b7b037386451317b5ea5ee2861ce46ae22aff66e1ef0a462b4f42
|
7
|
+
data.tar.gz: 19ff469baf507e141f91a17161d7bf2b48172e273b87b1ef1abb2464c385dc7ea7ced75e64baf04928295bce218203d5f48d2ecedd718efd7291ba53693c78c0
|
Binary file
|
data/classpath/{embulk-util-aws-credentials-0.2.10.jar → embulk-util-aws-credentials-0.2.11.jar}
RENAMED
Binary file
|
@@ -43,6 +43,9 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
|
|
43
43
|
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
44
44
|
import org.embulk.util.aws.credentials.AwsCredentials;
|
45
45
|
import org.embulk.util.aws.credentials.AwsCredentialsTask;
|
46
|
+
|
47
|
+
import static com.amazonaws.Protocol.HTTP;
|
48
|
+
import static com.amazonaws.Protocol.HTTPS;
|
46
49
|
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
47
50
|
|
48
51
|
public abstract class AbstractS3FileInputPlugin
|
@@ -67,6 +70,11 @@ public abstract class AbstractS3FileInputPlugin
|
|
67
70
|
@ConfigDefault("null")
|
68
71
|
public Optional<String> getAccessKeyId();
|
69
72
|
|
73
|
+
@Config("http_proxy")
|
74
|
+
@ConfigDefault("null")
|
75
|
+
public Optional<HttpProxy> getHttpProxy();
|
76
|
+
public void setHttpProxy(Optional<HttpProxy> httpProxy);
|
77
|
+
|
70
78
|
@Config("incremental")
|
71
79
|
@ConfigDefault("true")
|
72
80
|
public boolean getIncremental();
|
@@ -144,9 +152,38 @@ public abstract class AbstractS3FileInputPlugin
|
|
144
152
|
clientConfig.setMaxErrorRetry(3); // SDK default: 3
|
145
153
|
clientConfig.setSocketTimeout(8*60*1000); // SDK default: 50*1000
|
146
154
|
|
155
|
+
// set http proxy
|
156
|
+
if (task.getHttpProxy().isPresent()) {
|
157
|
+
setHttpProxyInAwsClient(clientConfig, task.getHttpProxy().get());
|
158
|
+
}
|
159
|
+
|
147
160
|
return clientConfig;
|
148
161
|
}
|
149
162
|
|
163
|
+
private void setHttpProxyInAwsClient(ClientConfiguration clientConfig, HttpProxy httpProxy)
|
164
|
+
{
|
165
|
+
// host
|
166
|
+
clientConfig.setProxyHost(httpProxy.getHost());
|
167
|
+
|
168
|
+
// port
|
169
|
+
if (httpProxy.getPort().isPresent()) {
|
170
|
+
clientConfig.setProxyPort(httpProxy.getPort().get());
|
171
|
+
}
|
172
|
+
|
173
|
+
// https
|
174
|
+
clientConfig.setProtocol(httpProxy.getHttps() ? Protocol.HTTPS : Protocol.HTTP);
|
175
|
+
|
176
|
+
// user
|
177
|
+
if (httpProxy.getUser().isPresent()) {
|
178
|
+
clientConfig.setProxyUsername(httpProxy.getUser().get());
|
179
|
+
}
|
180
|
+
|
181
|
+
// password
|
182
|
+
if (httpProxy.getPassword().isPresent()) {
|
183
|
+
clientConfig.setProxyPassword(httpProxy.getPassword().get());
|
184
|
+
}
|
185
|
+
}
|
186
|
+
|
150
187
|
private FileList listFiles(PluginTask task)
|
151
188
|
{
|
152
189
|
try {
|
@@ -0,0 +1,35 @@
|
|
1
|
+
package org.embulk.input.s3;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.config.Config;
|
5
|
+
import org.embulk.config.ConfigDefault;
|
6
|
+
import org.embulk.config.Task;
|
7
|
+
|
8
|
+
/**
|
9
|
+
* HttpProxy is config unit for Input/Output plugins' configs.
|
10
|
+
*
|
11
|
+
* TODO: This unit will be moved to embulk/embulk-plugin-units.git.
|
12
|
+
* TODO: Consider using @JsonProperty(defaultValue=...) in Jackson 2.6+.
|
13
|
+
*/
|
14
|
+
public interface HttpProxy
|
15
|
+
extends Task
|
16
|
+
{
|
17
|
+
@Config("host")
|
18
|
+
public String getHost();
|
19
|
+
|
20
|
+
@Config("port")
|
21
|
+
@ConfigDefault("null")
|
22
|
+
public Optional<Integer> getPort();
|
23
|
+
|
24
|
+
@Config("https")
|
25
|
+
@ConfigDefault("true")
|
26
|
+
public boolean getHttps();
|
27
|
+
|
28
|
+
@Config("user")
|
29
|
+
@ConfigDefault("null")
|
30
|
+
public Optional<String> getUser();
|
31
|
+
|
32
|
+
@Config("password")
|
33
|
+
@ConfigDefault("null")
|
34
|
+
public Optional<String> getPassword();
|
35
|
+
}
|
@@ -0,0 +1,120 @@
|
|
1
|
+
package org.embulk.input.s3;
|
2
|
+
|
3
|
+
import com.google.common.base.Optional;
|
4
|
+
import org.embulk.EmbulkTestRuntime;
|
5
|
+
import org.embulk.config.ConfigSource;
|
6
|
+
import org.embulk.input.s3.S3FileInputPlugin.S3PluginTask;
|
7
|
+
import org.junit.Before;
|
8
|
+
import org.junit.Rule;
|
9
|
+
import org.junit.Test;
|
10
|
+
|
11
|
+
import static org.junit.Assert.assertEquals;
|
12
|
+
import static org.junit.Assert.assertTrue;
|
13
|
+
|
14
|
+
public class TestHttpProxy
|
15
|
+
{
|
16
|
+
@Rule
|
17
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
18
|
+
|
19
|
+
private ConfigSource config;
|
20
|
+
|
21
|
+
@Before
|
22
|
+
public void createResources()
|
23
|
+
{
|
24
|
+
config = runtime.getExec().newConfigSource();
|
25
|
+
setupS3Config(config);
|
26
|
+
}
|
27
|
+
|
28
|
+
@Test
|
29
|
+
public void checkDefaultHttpProxy()
|
30
|
+
{
|
31
|
+
ConfigSource conf = config.deepCopy();
|
32
|
+
setupS3Config(conf);
|
33
|
+
S3PluginTask task = conf.loadConfig(S3PluginTask.class);
|
34
|
+
assertTrue(!task.getHttpProxy().isPresent());
|
35
|
+
}
|
36
|
+
|
37
|
+
@Test
|
38
|
+
public void checkHttpProxy()
|
39
|
+
{
|
40
|
+
{ // specify host
|
41
|
+
String host = "my_host";
|
42
|
+
ConfigSource conf = config.deepCopy().set("host", host);
|
43
|
+
HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
|
44
|
+
assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
|
45
|
+
httpProxy);
|
46
|
+
}
|
47
|
+
|
48
|
+
{ // specify https=true explicitly
|
49
|
+
String host = "my_host";
|
50
|
+
ConfigSource conf = config.deepCopy()
|
51
|
+
.set("host", host)
|
52
|
+
.set("https", true);
|
53
|
+
HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
|
54
|
+
assertHttpProxy(host, Optional.<Integer>absent(), true, Optional.<String>absent(), Optional.<String>absent(),
|
55
|
+
httpProxy);
|
56
|
+
}
|
57
|
+
|
58
|
+
{ // specify https=false
|
59
|
+
String host = "my_host";
|
60
|
+
ConfigSource conf = config.deepCopy()
|
61
|
+
.set("host", host)
|
62
|
+
.set("https", false);
|
63
|
+
HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
|
64
|
+
assertHttpProxy(host, Optional.<Integer>absent(), false, Optional.<String>absent(), Optional.<String>absent(),
|
65
|
+
httpProxy);
|
66
|
+
}
|
67
|
+
|
68
|
+
{ // specify host, port
|
69
|
+
String host = "my_host";
|
70
|
+
int port = 8080;
|
71
|
+
ConfigSource conf = config.deepCopy()
|
72
|
+
.set("host", host)
|
73
|
+
.set("port", port);
|
74
|
+
HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
|
75
|
+
assertHttpProxy(host, Optional.of(port), true, Optional.<String>absent(), Optional.<String>absent(),
|
76
|
+
httpProxy);
|
77
|
+
}
|
78
|
+
|
79
|
+
{ // specify host, port, user, password
|
80
|
+
String host = "my_host";
|
81
|
+
int port = 8080;
|
82
|
+
String user = "my_user";
|
83
|
+
String password = "my_pass";
|
84
|
+
ConfigSource conf = config.deepCopy()
|
85
|
+
.set("host", host)
|
86
|
+
.set("port", port)
|
87
|
+
.set("user", user)
|
88
|
+
.set("password", password);
|
89
|
+
HttpProxy httpProxy = conf.loadConfig(HttpProxy.class);
|
90
|
+
assertHttpProxy(host, Optional.of(port), true, Optional.of(user), Optional.of(password),
|
91
|
+
httpProxy);
|
92
|
+
}
|
93
|
+
}
|
94
|
+
|
95
|
+
private static void setupS3Config(ConfigSource config)
|
96
|
+
{
|
97
|
+
config.set("bucket", "my_bucket").set("path_prefix", "my_path_prefix");
|
98
|
+
}
|
99
|
+
|
100
|
+
private static void assertHttpProxy(String host, Optional<Integer> port, boolean https, Optional<String> user, Optional<String> password,
|
101
|
+
HttpProxy actual)
|
102
|
+
{
|
103
|
+
assertEquals(host, actual.getHost());
|
104
|
+
assertEquals(port.isPresent(), actual.getPort().isPresent());
|
105
|
+
if (port.isPresent()) {
|
106
|
+
assertEquals(port.get(), actual.getPort().get());
|
107
|
+
}
|
108
|
+
|
109
|
+
assertEquals(https, actual.getHttps());
|
110
|
+
|
111
|
+
assertEquals(user.isPresent(), actual.getUser().isPresent());
|
112
|
+
if (user.isPresent()) {
|
113
|
+
assertEquals(user.get(), actual.getUser().get());
|
114
|
+
}
|
115
|
+
assertEquals(password.isPresent(), actual.getPassword().isPresent());
|
116
|
+
if (password.isPresent()) {
|
117
|
+
assertEquals(password.get(), actual.getPassword().get());
|
118
|
+
}
|
119
|
+
}
|
120
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.11
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-04-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -49,9 +49,11 @@ files:
|
|
49
49
|
- lib/embulk/input/s3.rb
|
50
50
|
- src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
|
51
51
|
- src/main/java/org/embulk/input/s3/FileList.java
|
52
|
+
- src/main/java/org/embulk/input/s3/HttpProxy.java
|
52
53
|
- src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
|
53
54
|
- src/test/java/org/embulk/input/s3/TestAwsCredentials.java
|
54
55
|
- src/test/java/org/embulk/input/s3/TestFileList.java
|
56
|
+
- src/test/java/org/embulk/input/s3/TestHttpProxy.java
|
55
57
|
- src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
|
56
58
|
- src/test/java/org/embulk/input/s3/TestS3InputStreamReopener.java
|
57
59
|
- src/test/resources/sample_01.csv
|
@@ -59,8 +61,8 @@ files:
|
|
59
61
|
- classpath/aws-java-sdk-kms-1.10.33.jar
|
60
62
|
- classpath/aws-java-sdk-s3-1.10.33.jar
|
61
63
|
- classpath/commons-codec-1.6.jar
|
62
|
-
- classpath/embulk-input-s3-0.2.
|
63
|
-
- classpath/embulk-util-aws-credentials-0.2.
|
64
|
+
- classpath/embulk-input-s3-0.2.11.jar
|
65
|
+
- classpath/embulk-util-aws-credentials-0.2.11.jar
|
64
66
|
- classpath/httpclient-4.3.6.jar
|
65
67
|
- classpath/httpcore-4.3.3.jar
|
66
68
|
- classpath/jcl-over-slf4j-1.7.12.jar
|
Binary file
|