embulk-input-s3 0.2.13 → 0.2.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-input-s3-0.2.14.jar +0 -0
- data/classpath/{embulk-util-aws-credentials-0.2.13.jar → embulk-util-aws-credentials-0.2.14.jar} +0 -0
- data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +16 -2
- data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java +6 -3
- data/src/test/java/org/embulk/input/s3/TestS3FileInputPlugin.java +49 -1
- metadata +4 -4
- data/classpath/embulk-input-s3-0.2.13.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a69ebc7bdba28093d994a2e19842c108f265b1a0
|
4
|
+
data.tar.gz: 2ca38204ff6fc9a01d9032e7c6ceb0a3a38f4629
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ddc5c1c7f780bd1bc4cad5c1647795d7e25d1564d6d9611970ae86b519f3bb94546e06c6138bc4509df9173453bbeedb2bb31ea90b0759b7dada73b54a1050b9
|
7
|
+
data.tar.gz: d598b8687a1490cfb40d01ea3f730736ae4053848a6ffa630757aad63feab774caa3910e276819db52e58ed72b93a4eedfbe9f56cef7089d925787d77370f987
|
Binary file
|
data/classpath/{embulk-util-aws-credentials-0.2.13.jar → embulk-util-aws-credentials-0.2.14.jar}
RENAMED
Binary file
|
@@ -11,6 +11,7 @@ import com.amazonaws.services.s3.model.ListObjectsRequest;
|
|
11
11
|
import com.amazonaws.services.s3.model.ObjectListing;
|
12
12
|
import com.amazonaws.services.s3.model.S3Object;
|
13
13
|
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
14
|
+
import com.amazonaws.services.s3.model.StorageClass;
|
14
15
|
import com.google.common.annotations.VisibleForTesting;
|
15
16
|
import com.google.common.base.Optional;
|
16
17
|
import com.google.common.base.Throwables;
|
@@ -74,6 +75,10 @@ public abstract class AbstractS3FileInputPlugin
|
|
74
75
|
@ConfigDefault("true")
|
75
76
|
public boolean getIncremental();
|
76
77
|
|
78
|
+
@Config("skip_glacier_objects")
|
79
|
+
@ConfigDefault("false")
|
80
|
+
public boolean getSkipGlacierObjects();
|
81
|
+
|
77
82
|
// TODO timeout, ssl, etc
|
78
83
|
|
79
84
|
public FileList getFiles();
|
@@ -206,7 +211,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
206
211
|
|
207
212
|
FileList.Builder builder = new FileList.Builder(task);
|
208
213
|
listS3FilesByPrefix(builder, client, bucketName,
|
209
|
-
task.getPathPrefix(), task.getLastPath());
|
214
|
+
task.getPathPrefix(), task.getLastPath(), task.getSkipGlacierObjects());
|
210
215
|
return builder.build();
|
211
216
|
}
|
212
217
|
catch (AmazonServiceException ex) {
|
@@ -229,13 +234,22 @@ public abstract class AbstractS3FileInputPlugin
|
|
229
234
|
*/
|
230
235
|
public static void listS3FilesByPrefix(FileList.Builder builder,
|
231
236
|
AmazonS3 client, String bucketName,
|
232
|
-
String prefix, Optional<String> lastPath)
|
237
|
+
String prefix, Optional<String> lastPath, boolean skipGlacierObjects)
|
233
238
|
{
|
234
239
|
String lastKey = lastPath.orNull();
|
235
240
|
do {
|
236
241
|
ListObjectsRequest req = new ListObjectsRequest(bucketName, prefix, lastKey, null, 1024);
|
237
242
|
ObjectListing ol = client.listObjects(req);
|
238
243
|
for (S3ObjectSummary s : ol.getObjectSummaries()) {
|
244
|
+
if (s.getStorageClass().equals(StorageClass.Glacier.toString())) {
|
245
|
+
if (skipGlacierObjects) {
|
246
|
+
Exec.getLogger("AbstractS3FileInputPlugin.class").warn("Skipped \"s3://{}/{}\" that stored at Glacier.", bucketName, s.getKey());
|
247
|
+
continue;
|
248
|
+
}
|
249
|
+
else {
|
250
|
+
throw new ConfigException("Detected an object stored at Glacier. Set \"skip_glacier_objects\" option to \"true\" to skip this.");
|
251
|
+
}
|
252
|
+
}
|
239
253
|
if (s.getSize() > 0) {
|
240
254
|
builder.add(s.getKey(), s.getSize());
|
241
255
|
if (!builder.needsMore()) {
|
@@ -27,7 +27,8 @@ public class S3FileInputPlugin
|
|
27
27
|
private static final Logger log = Exec.getLogger(S3FileInputPlugin.class);
|
28
28
|
|
29
29
|
@Override
|
30
|
-
protected Class<? extends PluginTask> getTaskClass()
|
30
|
+
protected Class<? extends PluginTask> getTaskClass()
|
31
|
+
{
|
31
32
|
return S3PluginTask.class;
|
32
33
|
}
|
33
34
|
|
@@ -47,9 +48,11 @@ public class S3FileInputPlugin
|
|
47
48
|
"if both is specified only the endpoint will be in effect.");
|
48
49
|
}
|
49
50
|
builder.setEndpointConfiguration(new EndpointConfiguration(endpoint.get(), null));
|
50
|
-
}
|
51
|
+
}
|
52
|
+
else if (region.isPresent()) {
|
51
53
|
builder.setRegion(region.get());
|
52
|
-
}
|
54
|
+
}
|
55
|
+
else {
|
53
56
|
// This is to keep the AWS SDK upgrading to 1.11.x to be backward compatible with old configuration.
|
54
57
|
//
|
55
58
|
// On SDK 1.10.x, when neither endpoint nor region is set explicitly, the client's endpoint will be by
|
@@ -1,11 +1,17 @@
|
|
1
1
|
package org.embulk.input.s3;
|
2
2
|
|
3
3
|
import com.amazonaws.services.s3.AmazonS3;
|
4
|
+
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
5
|
+
import com.amazonaws.services.s3.model.ObjectListing;
|
4
6
|
import com.amazonaws.services.s3.model.Region;
|
7
|
+
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
8
|
+
import com.amazonaws.services.s3.model.StorageClass;
|
9
|
+
import com.google.common.base.Optional;
|
5
10
|
import com.google.common.collect.ImmutableList;
|
6
11
|
import com.google.common.collect.ImmutableMap;
|
7
12
|
import org.embulk.EmbulkTestRuntime;
|
8
13
|
import org.embulk.config.ConfigDiff;
|
14
|
+
import org.embulk.config.ConfigException;
|
9
15
|
import org.embulk.config.ConfigSource;
|
10
16
|
import org.embulk.config.TaskReport;
|
11
17
|
import org.embulk.config.TaskSource;
|
@@ -20,7 +26,9 @@ import org.junit.Before;
|
|
20
26
|
import org.junit.BeforeClass;
|
21
27
|
import org.junit.Rule;
|
22
28
|
import org.junit.Test;
|
29
|
+
import org.mockito.Mockito;
|
23
30
|
|
31
|
+
import java.lang.reflect.Field;
|
24
32
|
import java.util.ArrayList;
|
25
33
|
import java.util.List;
|
26
34
|
|
@@ -29,6 +37,9 @@ import static org.junit.Assert.assertEquals;
|
|
29
37
|
import static org.junit.Assert.assertFalse;
|
30
38
|
import static org.junit.Assert.assertNull;
|
31
39
|
import static org.junit.Assume.assumeNotNull;
|
40
|
+
import static org.mockito.Matchers.any;
|
41
|
+
import static org.mockito.Mockito.doReturn;
|
42
|
+
import static org.mockito.Mockito.mock;
|
32
43
|
|
33
44
|
public class TestS3FileInputPlugin
|
34
45
|
{
|
@@ -150,7 +161,6 @@ public class TestS3FileInputPlugin
|
|
150
161
|
}
|
151
162
|
}
|
152
163
|
|
153
|
-
|
154
164
|
@Test
|
155
165
|
public void configuredEndpoint()
|
156
166
|
{
|
@@ -195,6 +205,44 @@ public class TestS3FileInputPlugin
|
|
195
205
|
assertEquals(s3Client.getRegion(), Region.US_Standard);
|
196
206
|
}
|
197
207
|
|
208
|
+
@Test(expected = ConfigException.class)
|
209
|
+
public void useSkipGlacierObjects() throws Exception
|
210
|
+
{
|
211
|
+
AmazonS3 client;
|
212
|
+
client = mock(AmazonS3.class);
|
213
|
+
doReturn(s3objectList("in/aa/a", StorageClass.Glacier)).when(client).listObjects(any(ListObjectsRequest.class));
|
214
|
+
|
215
|
+
AbstractS3FileInputPlugin plugin = Mockito.mock(AbstractS3FileInputPlugin.class, Mockito.CALLS_REAL_METHODS);
|
216
|
+
plugin.listS3FilesByPrefix(newFileList(config, "sample_00", 100L), client, "test_bucket", "test_prefix", Optional.<String>absent(), false);
|
217
|
+
}
|
218
|
+
|
219
|
+
private FileList.Builder newFileList(ConfigSource config, Object... nameAndSize)
|
220
|
+
{
|
221
|
+
FileList.Builder builder = new FileList.Builder(config);
|
222
|
+
for (int i = 0; i < nameAndSize.length; i += 2) {
|
223
|
+
builder.add((String) nameAndSize[i], (long) nameAndSize[i + 1]);
|
224
|
+
}
|
225
|
+
return builder;
|
226
|
+
}
|
227
|
+
|
228
|
+
private ObjectListing s3objectList(String key, StorageClass storageClass) throws Exception
|
229
|
+
{
|
230
|
+
ObjectListing list = new ObjectListing();
|
231
|
+
|
232
|
+
S3ObjectSummary element = new S3ObjectSummary();
|
233
|
+
element.setKey(key);
|
234
|
+
element.setStorageClass(storageClass.toString());
|
235
|
+
|
236
|
+
List<S3ObjectSummary> objectSummaries = new ArrayList<>();
|
237
|
+
objectSummaries.add(element);
|
238
|
+
|
239
|
+
Field field = list.getClass().getDeclaredField("objectSummaries");
|
240
|
+
field.setAccessible(true);
|
241
|
+
field.set(list, objectSummaries);
|
242
|
+
|
243
|
+
return list;
|
244
|
+
}
|
245
|
+
|
198
246
|
static class Control
|
199
247
|
implements InputPlugin.Control
|
200
248
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-02-
|
11
|
+
date: 2018-02-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -61,8 +61,8 @@ files:
|
|
61
61
|
- classpath/aws-java-sdk-kms-1.11.253.jar
|
62
62
|
- classpath/aws-java-sdk-s3-1.11.253.jar
|
63
63
|
- classpath/commons-codec-1.9.jar
|
64
|
-
- classpath/embulk-input-s3-0.2.
|
65
|
-
- classpath/embulk-util-aws-credentials-0.2.
|
64
|
+
- classpath/embulk-input-s3-0.2.14.jar
|
65
|
+
- classpath/embulk-util-aws-credentials-0.2.14.jar
|
66
66
|
- classpath/httpclient-4.5.2.jar
|
67
67
|
- classpath/httpcore-4.4.4.jar
|
68
68
|
- classpath/ion-java-1.0.2.jar
|
Binary file
|