embulk-input-s3 0.2.19 → 0.2.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-input-s3-0.2.20.jar +0 -0
- data/classpath/{embulk-util-aws-credentials-0.2.19.jar → embulk-util-aws-credentials-0.2.20.jar} +0 -0
- data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +9 -7
- data/src/main/java/org/embulk/input/s3/{AlwaysRetryable.java → DefaultRetryable.java} +24 -8
- data/src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java +48 -0
- data/src/test/java/org/embulk/input/s3/{TestAlwaysRetryable.java → TestDefaultRetryable.java} +11 -11
- data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java +6 -3
- metadata +6 -6
- data/classpath/embulk-input-s3-0.2.19.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 0bd5f1404e192a97d82dba826a0c18294cb39fca
|
4
|
+
data.tar.gz: 37a0d8df4fde59f2214640f0e6b0fc41a4caf7e9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9912af96e7b5459b2bc92f2c2415b5e2b635ac9ef8ff70d411f7c929e4186a2f12c9bdafd1f173e4fc0913258fafad20b0ebc36704d45c99068ae076bf05fc68
|
7
|
+
data.tar.gz: def99ee22f5b8dcc65bc0b96cb0ee1314d54bf69e181d596fac4123e8fbf99e1f96e1bccb8a18f818d176e1b2f3de16571538e16bce327f13a2e57dba9e0d08f
|
Binary file
|
data/classpath/{embulk-util-aws-credentials-0.2.19.jar → embulk-util-aws-credentials-0.2.20.jar}
RENAMED
Binary file
|
@@ -4,6 +4,7 @@ import com.amazonaws.AmazonServiceException;
|
|
4
4
|
import com.amazonaws.ClientConfiguration;
|
5
5
|
import com.amazonaws.Protocol;
|
6
6
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
7
|
+
import com.amazonaws.retry.PredefinedRetryPolicies;
|
7
8
|
import com.amazonaws.services.s3.AmazonS3;
|
8
9
|
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
|
9
10
|
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
@@ -177,11 +178,12 @@ public abstract class AbstractS3FileInputPlugin
|
|
177
178
|
{
|
178
179
|
ClientConfiguration clientConfig = new ClientConfiguration();
|
179
180
|
|
181
|
+
/** PLT-9886: disable built-in retry*/
|
180
182
|
//clientConfig.setProtocol(Protocol.HTTP);
|
181
|
-
clientConfig.setMaxConnections(50); // SDK default: 50
|
182
|
-
clientConfig.setMaxErrorRetry(3); // SDK default: 3
|
183
|
-
clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
|
184
|
-
|
183
|
+
// clientConfig.setMaxConnections(50); // SDK default: 50
|
184
|
+
// clientConfig.setMaxErrorRetry(3); // SDK default: 3
|
185
|
+
// clientConfig.setSocketTimeout(8 * 60 * 1000); // SDK default: 50*1000
|
186
|
+
clientConfig.setRetryPolicy(PredefinedRetryPolicies.NO_RETRY_POLICY);
|
185
187
|
// set http proxy
|
186
188
|
if (task.getHttpProxy().isPresent()) {
|
187
189
|
setHttpProxyInAwsClient(clientConfig, task.getHttpProxy().get());
|
@@ -283,7 +285,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
283
285
|
{
|
284
286
|
final GetObjectMetadataRequest objectMetadataRequest = new GetObjectMetadataRequest(bucket, objectKey);
|
285
287
|
|
286
|
-
ObjectMetadata objectMetadata = new
|
288
|
+
ObjectMetadata objectMetadata = new DefaultRetryable<ObjectMetadata>("Looking up for a single object") {
|
287
289
|
@Override
|
288
290
|
public ObjectMetadata call()
|
289
291
|
{
|
@@ -337,7 +339,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
337
339
|
do {
|
338
340
|
final String finalLastKey = lastKey;
|
339
341
|
final ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, finalLastKey, null, 1024);
|
340
|
-
ObjectListing ol = new
|
342
|
+
ObjectListing ol = new DefaultRetryable<ObjectListing>("Listing objects") {
|
341
343
|
@Override
|
342
344
|
public ObjectListing call()
|
343
345
|
{
|
@@ -403,7 +405,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
403
405
|
log.warn(String.format("S3 read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
|
404
406
|
request.setRange(offset, contentLength - 1); // [first, last]
|
405
407
|
|
406
|
-
return new
|
408
|
+
return new DefaultRetryable<S3ObjectInputStream>("Opening the file") {
|
407
409
|
@Override
|
408
410
|
public S3ObjectInputStream call()
|
409
411
|
{
|
@@ -1,10 +1,14 @@
|
|
1
1
|
package org.embulk.input.s3;
|
2
2
|
|
3
|
+
import com.amazonaws.AmazonServiceException;
|
3
4
|
import com.google.common.base.Throwables;
|
5
|
+
import org.apache.http.HttpStatus;
|
4
6
|
import org.embulk.spi.Exec;
|
5
7
|
import org.embulk.spi.util.RetryExecutor;
|
6
8
|
import org.slf4j.Logger;
|
7
9
|
|
10
|
+
import java.util.HashSet;
|
11
|
+
import java.util.Set;
|
8
12
|
import java.util.concurrent.Callable;
|
9
13
|
|
10
14
|
import static java.lang.String.format;
|
@@ -12,20 +16,27 @@ import static org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
|
12
16
|
import static org.embulk.spi.util.RetryExecutor.Retryable;
|
13
17
|
|
14
18
|
/**
|
15
|
-
*
|
19
|
+
* Retryable utility, regardless the occurred exceptions,
|
16
20
|
* Also provide a default approach for exception propagation.
|
17
21
|
*/
|
18
|
-
class
|
22
|
+
class DefaultRetryable<T> implements Retryable<T>
|
19
23
|
{
|
20
|
-
private static final Logger log = Exec.getLogger(
|
21
|
-
|
24
|
+
private static final Logger log = Exec.getLogger(DefaultRetryable.class);
|
25
|
+
private static final Set<Integer> NONRETRYABLE_STATUS_CODES = new HashSet<Integer>(2);
|
26
|
+
private static final Set<String> NONRETRYABLE_ERROR_CODES = new HashSet<String>(1);
|
22
27
|
private String operationName;
|
23
28
|
private Callable<T> callable;
|
24
29
|
|
30
|
+
static {
|
31
|
+
NONRETRYABLE_STATUS_CODES.add(HttpStatus.SC_FORBIDDEN);
|
32
|
+
NONRETRYABLE_STATUS_CODES.add(HttpStatus.SC_METHOD_NOT_ALLOWED);
|
33
|
+
NONRETRYABLE_ERROR_CODES.add("ExpiredToken");
|
34
|
+
}
|
35
|
+
|
25
36
|
/**
|
26
37
|
* @param operationName the name that will be referred on logging
|
27
38
|
*/
|
28
|
-
public
|
39
|
+
public DefaultRetryable(String operationName)
|
29
40
|
{
|
30
41
|
this.operationName = operationName;
|
31
42
|
}
|
@@ -34,18 +45,18 @@ class AlwaysRetryable<T> implements Retryable<T>
|
|
34
45
|
* @param operationName the name that will be referred on logging
|
35
46
|
* @param callable the operation, either define this at construction time or override the call() method
|
36
47
|
*/
|
37
|
-
public
|
48
|
+
public DefaultRetryable(String operationName, Callable<T> callable)
|
38
49
|
{
|
39
50
|
this.operationName = operationName;
|
40
51
|
this.callable = callable;
|
41
52
|
}
|
42
53
|
|
43
|
-
public
|
54
|
+
public DefaultRetryable()
|
44
55
|
{
|
45
56
|
this("Anonymous operation");
|
46
57
|
}
|
47
58
|
|
48
|
-
public
|
59
|
+
public DefaultRetryable(Callable<T> callable)
|
49
60
|
{
|
50
61
|
this("Anonymous operation", callable);
|
51
62
|
}
|
@@ -64,6 +75,11 @@ class AlwaysRetryable<T> implements Retryable<T>
|
|
64
75
|
@Override
|
65
76
|
public boolean isRetryableException(Exception exception)
|
66
77
|
{
|
78
|
+
// No retry on a subset of service exceptions
|
79
|
+
if (exception instanceof AmazonServiceException) {
|
80
|
+
AmazonServiceException ase = (AmazonServiceException) exception;
|
81
|
+
return !NONRETRYABLE_STATUS_CODES.contains(ase.getStatusCode()) && !NONRETRYABLE_ERROR_CODES.contains(ase.getErrorCode());
|
82
|
+
}
|
67
83
|
return true;
|
68
84
|
}
|
69
85
|
|
@@ -1,11 +1,13 @@
|
|
1
1
|
package org.embulk.input.s3;
|
2
2
|
|
3
|
+
import com.amazonaws.AmazonServiceException;
|
3
4
|
import com.amazonaws.services.s3.AmazonS3;
|
4
5
|
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
|
5
6
|
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
6
7
|
import com.amazonaws.services.s3.model.ObjectListing;
|
7
8
|
import com.amazonaws.services.s3.model.ObjectMetadata;
|
8
9
|
import com.google.common.base.Optional;
|
10
|
+
import org.apache.http.HttpStatus;
|
9
11
|
import org.embulk.EmbulkTestRuntime;
|
10
12
|
import org.embulk.spi.util.RetryExecutor;
|
11
13
|
import org.junit.Before;
|
@@ -83,6 +85,52 @@ public class TestAbstractS3FileInputPlugin
|
|
83
85
|
retryExecutor().withRetryLimit(0));
|
84
86
|
}
|
85
87
|
|
88
|
+
@Test(expected = AmazonServiceException.class)
|
89
|
+
public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_forbidden_code()
|
90
|
+
{
|
91
|
+
AmazonServiceException exception = new AmazonServiceException("Forbidden exception");
|
92
|
+
exception.setStatusCode(HttpStatus.SC_FORBIDDEN);
|
93
|
+
exception.setErrorType(AmazonServiceException.ErrorType.Client);
|
94
|
+
|
95
|
+
doThrow(exception).doReturn(new ObjectListing())
|
96
|
+
.when(client).listObjects(any(ListObjectsRequest.class));
|
97
|
+
FileList.Builder builder = new FileList.Builder();
|
98
|
+
dummyS3Plugin().listS3FilesByPrefix(
|
99
|
+
builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
|
100
|
+
retryExecutor().withRetryLimit(1));
|
101
|
+
}
|
102
|
+
|
103
|
+
@Test(expected = AmazonServiceException.class)
|
104
|
+
public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_methodnotallow_code()
|
105
|
+
{
|
106
|
+
AmazonServiceException exception = new AmazonServiceException("method not allow exception");
|
107
|
+
exception.setStatusCode(HttpStatus.SC_METHOD_NOT_ALLOWED);
|
108
|
+
exception.setErrorType(AmazonServiceException.ErrorType.Client);
|
109
|
+
|
110
|
+
doThrow(exception).doReturn(new ObjectListing())
|
111
|
+
.when(client).listObjects(any(ListObjectsRequest.class));
|
112
|
+
FileList.Builder builder = new FileList.Builder();
|
113
|
+
dummyS3Plugin().listS3FilesByPrefix(
|
114
|
+
builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
|
115
|
+
retryExecutor().withRetryLimit(1));
|
116
|
+
}
|
117
|
+
|
118
|
+
@Test(expected = AmazonServiceException.class)
|
119
|
+
public void listS3FileByPrefix_on_retry_gave_up_should_throw_the_original_exception_in_expiredToken_code()
|
120
|
+
{
|
121
|
+
AmazonServiceException exception = new AmazonServiceException("expired token exception");
|
122
|
+
exception.setStatusCode(HttpStatus.SC_BAD_REQUEST);
|
123
|
+
exception.setErrorCode("ExpiredToken");
|
124
|
+
exception.setErrorType(AmazonServiceException.ErrorType.Client);
|
125
|
+
|
126
|
+
doThrow(exception).doReturn(new ObjectListing())
|
127
|
+
.when(client).listObjects(any(ListObjectsRequest.class));
|
128
|
+
FileList.Builder builder = new FileList.Builder();
|
129
|
+
dummyS3Plugin().listS3FilesByPrefix(
|
130
|
+
builder, client, "some_bucket", "some_prefix", Optional.of("last_path"), true,
|
131
|
+
retryExecutor().withRetryLimit(1));
|
132
|
+
}
|
133
|
+
|
86
134
|
@Test
|
87
135
|
public void addS3DirectObject()
|
88
136
|
{
|
data/src/test/java/org/embulk/input/s3/{TestAlwaysRetryable.java → TestDefaultRetryable.java}
RENAMED
@@ -12,10 +12,10 @@ import java.util.concurrent.Callable;
|
|
12
12
|
import static java.lang.String.format;
|
13
13
|
import static org.msgpack.core.Preconditions.checkArgument;
|
14
14
|
|
15
|
-
public class
|
15
|
+
public class TestDefaultRetryable
|
16
16
|
{
|
17
17
|
@Rule
|
18
|
-
public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); // require for
|
18
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime(); // require for DefaultRetryable's logger
|
19
19
|
|
20
20
|
private static class Deny extends RuntimeException implements Callable
|
21
21
|
{
|
@@ -71,16 +71,16 @@ public class TestAlwaysRetryable
|
|
71
71
|
{
|
72
72
|
retryExecutor()
|
73
73
|
.withRetryLimit(0)
|
74
|
-
.run(new
|
74
|
+
.run(new DefaultRetryable(Deny.until(0)));
|
75
75
|
retryExecutor()
|
76
76
|
.withRetryLimit(1)
|
77
|
-
.run(new
|
77
|
+
.run(new DefaultRetryable(Deny.until(1)));
|
78
78
|
retryExecutor()
|
79
79
|
.withRetryLimit(2)
|
80
|
-
.run(new
|
80
|
+
.run(new DefaultRetryable(Deny.until(1)));
|
81
81
|
retryExecutor()
|
82
82
|
.withRetryLimit(3)
|
83
|
-
.run(new
|
83
|
+
.run(new DefaultRetryable(Deny.until(2)));
|
84
84
|
}
|
85
85
|
|
86
86
|
@Test(expected = RetryGiveupException.class)
|
@@ -89,14 +89,14 @@ public class TestAlwaysRetryable
|
|
89
89
|
{
|
90
90
|
retryExecutor()
|
91
91
|
.withRetryLimit(3)
|
92
|
-
.run(new
|
92
|
+
.run(new DefaultRetryable(Deny.until(4)));
|
93
93
|
}
|
94
94
|
|
95
95
|
@Test(expected = Deny.class)
|
96
96
|
@SuppressWarnings("unchecked")
|
97
97
|
public void execute_should_unwrap_RetryGiveupException() throws Exception
|
98
98
|
{
|
99
|
-
new
|
99
|
+
new DefaultRetryable(Deny.until(4))
|
100
100
|
.executeWith(retryExecutor().withRetryLimit(3));
|
101
101
|
}
|
102
102
|
|
@@ -104,7 +104,7 @@ public class TestAlwaysRetryable
|
|
104
104
|
@SuppressWarnings("unchecked")
|
105
105
|
public void execute_should_unwrap_RetryGiveupException_but_rewrap_checked_exception_in_a_RuntimeException()
|
106
106
|
{
|
107
|
-
new
|
107
|
+
new DefaultRetryable(Deny.until(4).with(new Exception("A checked exception")))
|
108
108
|
.executeWith(retryExecutor().withRetryLimit(3));
|
109
109
|
}
|
110
110
|
|
@@ -114,7 +114,7 @@ public class TestAlwaysRetryable
|
|
114
114
|
RetryExecutor retryExc = retryExecutor().withRetryLimit(3);
|
115
115
|
// An explicit type parameter for operation return type is needed here,
|
116
116
|
// Without one, javac (at least on 1.8) will fails to infer the X exception type parameter.
|
117
|
-
new
|
117
|
+
new DefaultRetryable<Object>() {
|
118
118
|
@Override
|
119
119
|
public Object call() throws IOException
|
120
120
|
{
|
@@ -126,6 +126,6 @@ public class TestAlwaysRetryable
|
|
126
126
|
@Test(expected = IllegalStateException.class)
|
127
127
|
public void execute_without_an_implementation_should_throw_an_IllegalStateException()
|
128
128
|
{
|
129
|
-
new
|
129
|
+
new DefaultRetryable().executeWith(retryExecutor());
|
130
130
|
}
|
131
131
|
}
|
@@ -58,9 +58,12 @@ public class TestS3FileInputPlugin
|
|
58
58
|
@BeforeClass
|
59
59
|
public static void initializeConstantVariables()
|
60
60
|
{
|
61
|
-
EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET");
|
62
|
-
EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID");
|
63
|
-
EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY");
|
61
|
+
//EMBULK_S3_TEST_BUCKET = System.getenv("EMBULK_S3_TEST_BUCKET");
|
62
|
+
//EMBULK_S3_TEST_ACCESS_KEY_ID = System.getenv("EMBULK_S3_TEST_ACCESS_KEY_ID");
|
63
|
+
//EMBULK_S3_TEST_SECRET_ACCESS_KEY = System.getenv("EMBULK_S3_TEST_SECRET_ACCESS_KEY");
|
64
|
+
EMBULK_S3_TEST_BUCKET = "huytest2";
|
65
|
+
EMBULK_S3_TEST_ACCESS_KEY_ID="AKIAJ3EQVCVJIIM7RZDQ";
|
66
|
+
EMBULK_S3_TEST_SECRET_ACCESS_KEY="JeFvD40gosnbbBfY5B3Lxgoe96U5jIOkAF/e9Hh5";
|
64
67
|
assumeNotNull(EMBULK_S3_TEST_BUCKET, EMBULK_S3_TEST_ACCESS_KEY_ID, EMBULK_S3_TEST_SECRET_ACCESS_KEY);
|
65
68
|
}
|
66
69
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-07-23 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -48,14 +48,14 @@ files:
|
|
48
48
|
- build.gradle
|
49
49
|
- lib/embulk/input/s3.rb
|
50
50
|
- src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java
|
51
|
-
- src/main/java/org/embulk/input/s3/
|
51
|
+
- src/main/java/org/embulk/input/s3/DefaultRetryable.java
|
52
52
|
- src/main/java/org/embulk/input/s3/FileList.java
|
53
53
|
- src/main/java/org/embulk/input/s3/HttpProxy.java
|
54
54
|
- src/main/java/org/embulk/input/s3/RetrySupportPluginTask.java
|
55
55
|
- src/main/java/org/embulk/input/s3/S3FileInputPlugin.java
|
56
56
|
- src/test/java/org/embulk/input/s3/TestAbstractS3FileInputPlugin.java
|
57
|
-
- src/test/java/org/embulk/input/s3/TestAlwaysRetryable.java
|
58
57
|
- src/test/java/org/embulk/input/s3/TestAwsCredentials.java
|
58
|
+
- src/test/java/org/embulk/input/s3/TestDefaultRetryable.java
|
59
59
|
- src/test/java/org/embulk/input/s3/TestFileList.java
|
60
60
|
- src/test/java/org/embulk/input/s3/TestHttpProxy.java
|
61
61
|
- src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java
|
@@ -65,8 +65,8 @@ files:
|
|
65
65
|
- classpath/aws-java-sdk-kms-1.11.253.jar
|
66
66
|
- classpath/aws-java-sdk-s3-1.11.253.jar
|
67
67
|
- classpath/commons-codec-1.9.jar
|
68
|
-
- classpath/embulk-input-s3-0.2.
|
69
|
-
- classpath/embulk-util-aws-credentials-0.2.
|
68
|
+
- classpath/embulk-input-s3-0.2.20.jar
|
69
|
+
- classpath/embulk-util-aws-credentials-0.2.20.jar
|
70
70
|
- classpath/httpclient-4.5.2.jar
|
71
71
|
- classpath/httpcore-4.4.4.jar
|
72
72
|
- classpath/ion-java-1.0.2.jar
|
Binary file
|