embulk-input-s3 0.2.14 → 0.2.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/classpath/embulk-input-s3-0.2.15.jar +0 -0
- data/classpath/{embulk-util-aws-credentials-0.2.14.jar → embulk-util-aws-credentials-0.2.15.jar} +0 -0
- data/src/main/java/org/embulk/input/s3/AbstractS3FileInputPlugin.java +14 -6
- metadata +4 -4
- data/classpath/embulk-input-s3-0.2.14.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6efa3b427ccee34a13fae779d053ea5863a676e7
|
4
|
+
data.tar.gz: a54eec2a80f7003d4a7bdeff529e152fe8a7ed3b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 35d00e37563f72c184f1bf9f02186927975a14d9eae0bf2ea240fe31c94b28414b70c1574494be19833305a9908e72081a1b5356da570b48d85106366666cdd3
|
7
|
+
data.tar.gz: 7de346f325126fbc1d1ebd5ca7f36269945bc5d7e9227d4b6bfb88f7b0c56327ed2c95b902377b5c84e297898b9e98a6f14021b0634a0da8c18aed23f8d96c63
|
Binary file
|
data/classpath/{embulk-util-aws-credentials-0.2.14.jar → embulk-util-aws-credentials-0.2.15.jar}
RENAMED
Binary file
|
@@ -47,7 +47,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
|
47
47
|
public abstract class AbstractS3FileInputPlugin
|
48
48
|
implements FileInputPlugin
|
49
49
|
{
|
50
|
-
private final Logger
|
50
|
+
private static final Logger LOGGER = Exec.getLogger(S3FileInputPlugin.class);
|
51
51
|
|
52
52
|
public interface PluginTask
|
53
53
|
extends AwsCredentialsTask, FileList.Task, Task
|
@@ -119,9 +119,10 @@ public abstract class AbstractS3FileInputPlugin
|
|
119
119
|
|
120
120
|
// last_path
|
121
121
|
if (task.getIncremental()) {
|
122
|
-
|
122
|
+
Optional<String> lastPath = task.getFiles().getLastPath(task.getLastPath());
|
123
|
+
LOGGER.info("Incremental job, setting last_path to [{}]", lastPath.orNull());
|
124
|
+
configDiff.set("last_path", lastPath);
|
123
125
|
}
|
124
|
-
|
125
126
|
return configDiff;
|
126
127
|
}
|
127
128
|
|
@@ -201,17 +202,19 @@ public abstract class AbstractS3FileInputPlugin
|
|
201
202
|
|
202
203
|
private FileList listFiles(PluginTask task)
|
203
204
|
{
|
205
|
+
LOGGER.info("Start listing file with prefix [{}]", task.getPathPrefix());
|
204
206
|
try {
|
205
207
|
AmazonS3 client = newS3Client(task);
|
206
208
|
String bucketName = task.getBucket();
|
207
209
|
|
208
210
|
if (task.getPathPrefix().equals("/")) {
|
209
|
-
|
211
|
+
LOGGER.info("Listing files with prefix \"/\". This doesn't mean all files in a bucket. If you intend to read all files, use \"path_prefix: ''\" (empty string) instead.");
|
210
212
|
}
|
211
213
|
|
212
214
|
FileList.Builder builder = new FileList.Builder(task);
|
213
215
|
listS3FilesByPrefix(builder, client, bucketName,
|
214
216
|
task.getPathPrefix(), task.getLastPath(), task.getSkipGlacierObjects());
|
217
|
+
LOGGER.info("Found total [{}] files", builder.size());
|
215
218
|
return builder.build();
|
216
219
|
}
|
217
220
|
catch (AmazonServiceException ex) {
|
@@ -253,6 +256,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
253
256
|
if (s.getSize() > 0) {
|
254
257
|
builder.add(s.getKey(), s.getSize());
|
255
258
|
if (!builder.needsMore()) {
|
259
|
+
LOGGER.warn("Too many files matched, stop listing file");
|
256
260
|
return;
|
257
261
|
}
|
258
262
|
}
|
@@ -326,6 +330,7 @@ public abstract class AbstractS3FileInputPlugin
|
|
326
330
|
public void onGiveup(Exception firstException, Exception lastException)
|
327
331
|
throws RetryGiveupException
|
328
332
|
{
|
333
|
+
log.error("Giving up retry, first exception is [{}], last exception is [{}]", firstException.getMessage(), lastException.getMessage());
|
329
334
|
}
|
330
335
|
});
|
331
336
|
}
|
@@ -384,9 +389,12 @@ public abstract class AbstractS3FileInputPlugin
|
|
384
389
|
if (!iterator.hasNext()) {
|
385
390
|
return null;
|
386
391
|
}
|
387
|
-
|
392
|
+
String key = iterator.next();
|
393
|
+
GetObjectRequest request = new GetObjectRequest(bucket, key);
|
388
394
|
S3Object obj = client.getObject(request);
|
389
|
-
|
395
|
+
long objectSize = obj.getObjectMetadata().getContentLength();
|
396
|
+
LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize);
|
397
|
+
return new ResumableInputStream(obj.getObjectContent(), new S3InputStreamReopener(client, request, objectSize));
|
390
398
|
}
|
391
399
|
|
392
400
|
@Override
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-05-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -61,8 +61,8 @@ files:
|
|
61
61
|
- classpath/aws-java-sdk-kms-1.11.253.jar
|
62
62
|
- classpath/aws-java-sdk-s3-1.11.253.jar
|
63
63
|
- classpath/commons-codec-1.9.jar
|
64
|
-
- classpath/embulk-input-s3-0.2.
|
65
|
-
- classpath/embulk-util-aws-credentials-0.2.
|
64
|
+
- classpath/embulk-input-s3-0.2.15.jar
|
65
|
+
- classpath/embulk-util-aws-credentials-0.2.15.jar
|
66
66
|
- classpath/httpclient-4.5.2.jar
|
67
67
|
- classpath/httpcore-4.4.4.jar
|
68
68
|
- classpath/ion-java-1.0.2.jar
|
Binary file
|