embulk-input-s3 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +9 -0
- data/build.gradle +3 -3
- data/src/main/java/org/embulk/input/s3/S3FileInputPlugin.java +22 -40
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 83d9c51fe52f61880b83a5d97238ad16858eca69
|
4
|
+
data.tar.gz: 11b3a0a14d2606f93bb3dff5b3c469d75471dbaa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 34cd105f626902545a8250f2e6a5bd48e8d3b5c898e9df8112e333d6a6e51ea69bfa8d081175e6f07b1a9dc8d37b71ab6c728ea21c0188b9d1110fa6ab22a069
|
7
|
+
data.tar.gz: e1b82af62b79634ea454d114603ab6e5721bf0c01e1abe4e1ddbfead7fab2c7478a2a62c0c70fcbf839d24d9d2b03362bc3f9ea4b1c2f976576848ebf85dd122
|
data/ChangeLog
ADDED
data/build.gradle
CHANGED
@@ -13,11 +13,11 @@ configurations {
|
|
13
13
|
provided
|
14
14
|
}
|
15
15
|
|
16
|
-
version = "0.1.
|
16
|
+
version = "0.1.2"
|
17
17
|
|
18
18
|
dependencies {
|
19
|
-
compile "org.embulk:embulk-core:0.4.
|
20
|
-
provided "org.embulk:embulk-core:0.4.
|
19
|
+
compile "org.embulk:embulk-core:0.4.6"
|
20
|
+
provided "org.embulk:embulk-core:0.4.6"
|
21
21
|
compile "com.amazonaws:aws-java-sdk-s3:1.9.17"
|
22
22
|
testCompile "junit:junit:4.+"
|
23
23
|
testCompile "org.mockito:mockito-core:1.+"
|
@@ -1,40 +1,13 @@
|
|
1
1
|
package org.embulk.input.s3;
|
2
2
|
|
3
3
|
import java.util.List;
|
4
|
-
import java.
|
5
|
-
import java.
|
6
|
-
import java.io.InputStream;
|
7
|
-
import java.io.IOException;
|
8
|
-
import java.nio.file.Path;
|
9
|
-
import java.nio.file.Paths;
|
10
|
-
import java.nio.file.Files;
|
11
|
-
import java.nio.file.SimpleFileVisitor;
|
12
|
-
import java.nio.file.FileVisitResult;
|
13
|
-
import java.nio.file.attribute.BasicFileAttributes;
|
14
|
-
import javax.validation.constraints.NotNull;
|
15
|
-
import com.google.common.collect.ImmutableList;
|
16
|
-
import org.embulk.config.Config;
|
17
|
-
import org.embulk.config.Task;
|
18
|
-
import org.embulk.config.TaskSource;
|
19
|
-
import org.embulk.config.ConfigSource;
|
20
|
-
import org.embulk.config.ConfigDiff;
|
21
|
-
import org.embulk.config.ConfigInject;
|
22
|
-
import org.embulk.config.CommitReport;
|
23
|
-
import org.embulk.spi.BufferAllocator;
|
24
|
-
import org.embulk.spi.Exec;
|
25
|
-
import org.embulk.spi.FileInputPlugin;
|
26
|
-
import org.embulk.spi.TransactionalFileInput;
|
27
|
-
import org.embulk.spi.util.InputStreamFileInput;
|
28
|
-
import org.slf4j.Logger;
|
29
|
-
|
30
|
-
import static org.embulk.spi.util.Inputs.formatPath;
|
31
|
-
|
32
|
-
import java.util.List;
|
4
|
+
import java.util.ArrayList;
|
5
|
+
import java.util.Collections;
|
33
6
|
import java.io.IOException;
|
34
7
|
import java.io.InputStream;
|
35
8
|
import com.google.common.collect.ImmutableList;
|
36
9
|
import com.google.common.base.Optional;
|
37
|
-
import
|
10
|
+
import org.slf4j.Logger;
|
38
11
|
import com.amazonaws.auth.AWSCredentials;
|
39
12
|
import com.amazonaws.auth.AWSCredentialsProvider;
|
40
13
|
import com.amazonaws.auth.BasicAWSCredentials;
|
@@ -47,6 +20,7 @@ import com.amazonaws.services.s3.model.S3Object;
|
|
47
20
|
import com.amazonaws.ClientConfiguration;
|
48
21
|
import com.amazonaws.Protocol;
|
49
22
|
import org.embulk.config.Config;
|
23
|
+
import org.embulk.config.ConfigInject;
|
50
24
|
import org.embulk.config.ConfigDefault;
|
51
25
|
import org.embulk.config.Task;
|
52
26
|
import org.embulk.config.TaskSource;
|
@@ -104,22 +78,30 @@ public class S3FileInputPlugin
|
|
104
78
|
// list files recursively
|
105
79
|
task.setFiles(listFiles(task));
|
106
80
|
|
81
|
+
// TODO what if task.getFiles().isEmpty()?
|
82
|
+
|
107
83
|
// number of processors is same with number of files
|
108
84
|
return resume(task.dump(), task.getFiles().size(), control);
|
109
85
|
}
|
110
86
|
|
111
87
|
@Override
|
112
88
|
public ConfigDiff resume(TaskSource taskSource,
|
113
|
-
int
|
89
|
+
int taskCount,
|
114
90
|
FileInputPlugin.Control control)
|
115
91
|
{
|
116
|
-
|
117
|
-
|
92
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
93
|
+
|
94
|
+
control.run(taskSource, taskCount);
|
95
|
+
|
96
|
+
List<String> files = new ArrayList<String>(task.getFiles());
|
97
|
+
Collections.sort(files);
|
98
|
+
return Exec.newConfigDiff().
|
99
|
+
set("last_path", files.get(files.size() - 1));
|
118
100
|
}
|
119
101
|
|
120
102
|
@Override
|
121
103
|
public void cleanup(TaskSource taskSource,
|
122
|
-
int
|
104
|
+
int taskCount,
|
123
105
|
List<CommitReport> successCommitReports)
|
124
106
|
{
|
125
107
|
// do nothing
|
@@ -201,10 +183,10 @@ public class S3FileInputPlugin
|
|
201
183
|
}
|
202
184
|
|
203
185
|
@Override
|
204
|
-
public TransactionalFileInput open(TaskSource taskSource, int
|
186
|
+
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
205
187
|
{
|
206
188
|
PluginTask task = taskSource.loadTask(PluginTask.class);
|
207
|
-
return new S3FileInput(task,
|
189
|
+
return new S3FileInput(task, taskIndex);
|
208
190
|
}
|
209
191
|
|
210
192
|
public static class S3FileInput
|
@@ -220,11 +202,11 @@ public class S3FileInputPlugin
|
|
220
202
|
private final String key;
|
221
203
|
private boolean opened = false;
|
222
204
|
|
223
|
-
public SingleFileProvider(PluginTask task, int
|
205
|
+
public SingleFileProvider(PluginTask task, int taskIndex)
|
224
206
|
{
|
225
207
|
this.client = newS3Client(task);
|
226
208
|
this.bucket = task.getBucket();
|
227
|
-
this.key = task.getFiles().get(
|
209
|
+
this.key = task.getFiles().get(taskIndex);
|
228
210
|
}
|
229
211
|
|
230
212
|
@Override
|
@@ -250,9 +232,9 @@ public class S3FileInputPlugin
|
|
250
232
|
public void close() { }
|
251
233
|
}
|
252
234
|
|
253
|
-
public S3FileInput(PluginTask task, int
|
235
|
+
public S3FileInput(PluginTask task, int taskIndex)
|
254
236
|
{
|
255
|
-
super(task.getBufferAllocator(), new SingleFileProvider(task,
|
237
|
+
super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
|
256
238
|
}
|
257
239
|
|
258
240
|
public void abort() { }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-02-
|
11
|
+
date: 2015-02-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -60,6 +60,7 @@ extensions: []
|
|
60
60
|
extra_rdoc_files: []
|
61
61
|
files:
|
62
62
|
- .gitignore
|
63
|
+
- ChangeLog
|
63
64
|
- README.md
|
64
65
|
- build.gradle
|
65
66
|
- gradle/wrapper/gradle-wrapper.jar
|
@@ -74,7 +75,7 @@ files:
|
|
74
75
|
- classpath/aws-java-sdk-s3-1.9.17.jar
|
75
76
|
- classpath/commons-codec-1.6.jar
|
76
77
|
- classpath/commons-logging-1.1.3.jar
|
77
|
-
- classpath/embulk-input-s3-0.1.
|
78
|
+
- classpath/embulk-input-s3-0.1.2.jar
|
78
79
|
- classpath/httpclient-4.3.4.jar
|
79
80
|
- classpath/httpcore-4.3.2.jar
|
80
81
|
homepage: https://github.com/embulk/embulk-input-s3
|