embulk-output-s3 0.1.0 → 0.2.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ec5fe08d2ad38fad94f03679479e3653cfac20cb
4
- data.tar.gz: 6d363a602b57f9b9315b68a16602ca6a0af294c6
3
+ metadata.gz: 9da3670f4971d655c13d7a81cd908cc510a86b6f
4
+ data.tar.gz: 7e0bb6615000fd7a091057ccb33b385d3773e729
5
5
  SHA512:
6
- metadata.gz: bd0d1aa10571f12e130040a5073a5b7fcc649576f81f5bb83f8f32e12aba031fa56061a8c78d1308cf52445921e50754ab4490ffae2cdfa0dd49a3990003247e
7
- data.tar.gz: df38d944942edb611ff64ac85a243246a61c31ab03350baa811fd3f91fef77cc3f77b3024860f8ed9c73b4415b12c4aeb8ccf345e93f05db316cf8021de0e6f6
6
+ metadata.gz: 7cc3cf1287546ba8a40d69114146146ae1031ee5786789f072a3e80b6e1a682d0eabfe6504ac2abde1ac59deef0b557ccafe74899db8efb4dad6ccb1a1522028
7
+ data.tar.gz: 46e4f2a4a1a5b958f09e93e9ffc7299bfa6b0afbe6c5d7293b56fcb6083fb4b2a86019dcd15c29a27db3fbce9a9b105051f1f4ed2fb8a79702f2302672e5143b
data/.gitignore CHANGED
@@ -1,11 +1,11 @@
1
- *~
2
- /pkg/
3
- /tmp/
4
- .gradle/
5
- /classpath/
6
- build/
7
- .idea
8
- .settings/
9
- bin/
10
- .classpath
11
- .project
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ .gradle/
5
+ /classpath/
6
+ build/
7
+ .idea
8
+ .settings/
9
+ bin/
10
+ .classpath
11
+ .project
data/README.md CHANGED
@@ -1,39 +1,39 @@
1
- # S3 file output plugin for Embulk
2
-
3
- ## Overview
4
-
5
- * **Plugin type**: file output
6
- * **Load all or nothing**: no
7
- * **Resume supported**: yes
8
- * **Cleanup supported**: yes
9
-
10
- ## Configuration
11
-
12
- - **path_prefix**: prefix of target keys (string, required)
13
- - **file_ext**: suffix of target keys (string, required)
14
- - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
- - **bucket**: S3 bucket name (string, required)
16
- - **endpoint**: S3 endpoint login user name (string, optional)
17
- - **access_key_id**: AWS access key id (string, required)
18
- - **secret_access_key**: AWS secret key (string, required)
19
- - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
-
21
- ## Example
22
-
23
- ```yaml
24
- path_prefix: logs/out
25
- file_ext: .csv
26
- bucket: my-s3-bucket
27
- endpoint: s3-us-west-1.amazonaws.com
28
- access_key_id: ABCXYZ123ABCXYZ123
29
- secret_access_key: AbCxYz123aBcXyZ123
30
- formatter:
31
- type: csv
32
- ```
33
-
34
-
35
- ## Build
36
-
37
- ```
38
- $ ./gradlew gem
39
- ```
1
+ # S3 file output plugin for Embulk
2
+
3
+ ## Overview
4
+
5
+ * **Plugin type**: file output
6
+ * **Load all or nothing**: no
7
+ * **Resume supported**: yes
8
+ * **Cleanup supported**: yes
9
+
10
+ ## Configuration
11
+
12
+ - **path_prefix**: prefix of target keys (string, required)
13
+ - **file_ext**: suffix of target keys (string, required)
14
+ - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
+ - **bucket**: S3 bucket name (string, required)
16
+ - **endpoint**: S3 endpoint login user name (string, optional)
17
+ - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
18
+ - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
19
+ - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
+
21
+ ## Example
22
+
23
+ ```yaml
24
+ path_prefix: logs/out
25
+ file_ext: .csv
26
+ bucket: my-s3-bucket
27
+ endpoint: s3-us-west-1.amazonaws.com
28
+ access_key_id: ABCXYZ123ABCXYZ123
29
+ secret_access_key: AbCxYz123aBcXyZ123
30
+ formatter:
31
+ type: csv
32
+ ```
33
+
34
+
35
+ ## Build
36
+
37
+ ```
38
+ $ ./gradlew gem
39
+ ```
data/build.gradle CHANGED
@@ -1,56 +1,56 @@
1
- plugins {
2
- id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
4
- id "java"
5
- }
6
- import com.github.jrubygradle.JRubyExec
7
- repositories {
8
- mavenCentral()
9
- jcenter()
10
- }
11
- configurations {
12
- provided
13
- }
14
-
15
- version = "0.1.0"
16
-
17
- dependencies {
18
- compile "org.embulk:embulk-core:0.5.2"
19
- provided "org.embulk:embulk-core:0.5.2"
20
- compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
21
- testCompile "junit:junit:4.+"
22
- }
23
-
24
- task classpath(type: Copy, dependsOn: ["jar"]) {
25
- doFirst { file("classpath").deleteDir() }
26
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
27
- into "classpath"
28
- }
29
- clean { delete 'classpath' }
30
-
31
- task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
32
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
33
- script "build/gemspec"
34
- doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
35
- }
36
-
37
- task gemspec << { file("build/gemspec").write($/
38
- Gem::Specification.new do |spec|
39
- spec.name = "${project.name}"
40
- spec.version = "${project.version}"
41
- spec.authors = ["Manabu Takayama"]
42
- spec.summary = %[S3 file output plugin for Embulk]
43
- spec.description = %["Stores files on S3."]
44
- spec.email = ["learn.libra@gmail.com"]
45
- spec.licenses = ["MIT"]
46
- spec.homepage = "https://github.com/llibra/embulk-output-s3"
47
-
48
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
49
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
50
- spec.require_paths = ["lib"]
51
-
52
- spec.add_development_dependency 'bundler', ['~> 1.0']
53
- spec.add_development_dependency 'rake', ['>= 10.0']
54
- end
55
- /$)
56
- }
1
+ plugins {
2
+ id "com.jfrog.bintray" version "1.1"
3
+ id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "java"
5
+ }
6
+ import com.github.jrubygradle.JRubyExec
7
+ repositories {
8
+ mavenCentral()
9
+ jcenter()
10
+ }
11
+ configurations {
12
+ provided
13
+ }
14
+
15
+ version = "0.2.0"
16
+
17
+ dependencies {
18
+ compile "org.embulk:embulk-core:0.5.2"
19
+ provided "org.embulk:embulk-core:0.5.2"
20
+ compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
21
+ testCompile "junit:junit:4.+"
22
+ }
23
+
24
+ task classpath(type: Copy, dependsOn: ["jar"]) {
25
+ doFirst { file("classpath").deleteDir() }
26
+ from (configurations.runtime - configurations.provided + files(jar.archivePath))
27
+ into "classpath"
28
+ }
29
+ clean { delete 'classpath' }
30
+
31
+ task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
32
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
33
+ script "build/gemspec"
34
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
35
+ }
36
+
37
+ task gemspec << { file("build/gemspec").write($/
38
+ Gem::Specification.new do |spec|
39
+ spec.name = "${project.name}"
40
+ spec.version = "${project.version}"
41
+ spec.authors = ["Manabu Takayama"]
42
+ spec.summary = %[S3 file output plugin for Embulk]
43
+ spec.description = %[Stores files on S3.]
44
+ spec.email = ["learn.libra@gmail.com"]
45
+ spec.licenses = ["MIT"]
46
+ spec.homepage = "https://github.com/llibra/embulk-output-s3"
47
+
48
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
49
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
50
+ spec.require_paths = ["lib"]
51
+
52
+ spec.add_development_dependency 'bundler', ['~> 1.0']
53
+ spec.add_development_dependency 'rake', ['>= 10.0']
54
+ end
55
+ /$)
56
+ }
Binary file
@@ -1,258 +1,259 @@
1
- package org.embulk.output;
2
-
3
- import java.io.IOException;
4
- import java.io.OutputStream;
5
- import java.nio.file.Files;
6
- import java.nio.file.Path;
7
- import java.util.IllegalFormatException;
8
- import java.util.List;
9
- import java.util.Locale;
10
-
11
- import org.embulk.config.CommitReport;
12
- import org.embulk.config.Config;
13
- import org.embulk.config.ConfigDefault;
14
- import org.embulk.config.ConfigDiff;
15
- import org.embulk.config.ConfigException;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskSource;
19
- import org.embulk.spi.Buffer;
20
- import org.embulk.spi.Exec;
21
- import org.embulk.spi.FileOutput;
22
- import org.embulk.spi.FileOutputPlugin;
23
- import org.embulk.spi.TransactionalFileOutput;
24
- import org.slf4j.Logger;
25
-
26
- import com.amazonaws.ClientConfiguration;
27
- import com.amazonaws.auth.AWSCredentials;
28
- import com.amazonaws.auth.AWSCredentialsProvider;
29
- import com.amazonaws.auth.BasicAWSCredentials;
30
- import com.amazonaws.services.s3.AmazonS3Client;
31
- import com.amazonaws.services.s3.model.PutObjectRequest;
32
-
33
- public class S3FileOutputPlugin implements FileOutputPlugin {
34
- public interface PluginTask extends Task {
35
- @Config("path_prefix")
36
- public String getPathPrefix();
37
-
38
- @Config("file_ext")
39
- public String getFileNameExtension();
40
-
41
- @Config("sequence_format")
42
- @ConfigDefault("\".%03d.%02d\"")
43
- public String getSequenceFormat();
44
-
45
- @Config("bucket")
46
- public String getBucket();
47
-
48
- @Config("endpoint")
49
- public String getEndpoint();
50
-
51
- @Config("access_key_id")
52
- public String getAccessKeyId();
53
-
54
- @Config("secret_access_key")
55
- public String getSecretAccessKey();
56
-
57
- @Config("tmp_path_prefix")
58
- @ConfigDefault("\"embulk-output-s3-\"")
59
- public String getTempPathPrefix();
60
- }
61
-
62
- public static class S3FileOutput implements FileOutput,
63
- TransactionalFileOutput {
64
- private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
65
-
66
- private final String bucket;
67
- private final String pathPrefix;
68
- private final String sequenceFormat;
69
- private final String fileNameExtension;
70
- private final String tempPathPrefix;
71
-
72
- private int taskIndex;
73
- private int fileIndex;
74
- private AmazonS3Client client;
75
- private OutputStream current;
76
- private Path tempFilePath;
77
-
78
- public static AWSCredentialsProvider getCredentialsProvider(
79
- PluginTask task) {
80
- final AWSCredentials cred = new BasicAWSCredentials(
81
- task.getAccessKeyId(), task.getSecretAccessKey());
82
- return new AWSCredentialsProvider() {
83
- @Override
84
- public AWSCredentials getCredentials() {
85
- return cred;
86
- }
87
-
88
- @Override
89
- public void refresh() {
90
- }
91
- };
92
- }
93
-
94
- private static AmazonS3Client newS3Client(PluginTask task) {
95
- AWSCredentialsProvider credentials = getCredentialsProvider(task);
96
-
97
- ClientConfiguration config = new ClientConfiguration();
98
- // TODO: Support more configurations.
99
-
100
- AmazonS3Client client = new AmazonS3Client(credentials, config);
101
- client.setEndpoint(task.getEndpoint());
102
-
103
- return client;
104
- }
105
-
106
- public S3FileOutput(PluginTask task, int taskIndex) {
107
- this.taskIndex = taskIndex;
108
- this.client = newS3Client(task);
109
- this.bucket = task.getBucket();
110
- this.pathPrefix = task.getPathPrefix();
111
- this.sequenceFormat = task.getSequenceFormat();
112
- this.fileNameExtension = task.getFileNameExtension();
113
- this.tempPathPrefix = task.getTempPathPrefix();
114
- }
115
-
116
- private static Path newTempFile(String prefix) throws IOException {
117
- return Files.createTempFile(prefix, null);
118
- }
119
-
120
- private void deleteTempFile() {
121
- if (tempFilePath == null) {
122
- return;
123
- }
124
-
125
- try {
126
- Files.delete(tempFilePath);
127
- tempFilePath = null;
128
- } catch (IOException e) {
129
- throw new RuntimeException(e);
130
- }
131
- }
132
-
133
- private String buildCurrentKey() {
134
- String sequence = String.format(sequenceFormat, taskIndex,
135
- fileIndex);
136
- return pathPrefix + sequence + fileNameExtension;
137
- }
138
-
139
- private void putFile(Path from, String key) {
140
- PutObjectRequest request = new PutObjectRequest(bucket, key,
141
- from.toFile());
142
- client.putObject(request);
143
- }
144
-
145
- private void closeCurrent() {
146
- if (current == null) {
147
- return;
148
- }
149
-
150
- try {
151
- putFile(tempFilePath, buildCurrentKey());
152
- fileIndex++;
153
- } finally {
154
- try {
155
- current.close();
156
- current = null;
157
- } catch (IOException e) {
158
- throw new RuntimeException(e);
159
- } finally {
160
- deleteTempFile();
161
- }
162
- }
163
- }
164
-
165
- @Override
166
- public void nextFile() {
167
- closeCurrent();
168
-
169
- try {
170
- tempFilePath = newTempFile(tempPathPrefix);
171
-
172
- log.info("Writing S3 file '{}'", buildCurrentKey());
173
-
174
- current = Files.newOutputStream(tempFilePath);
175
- } catch (IOException e) {
176
- throw new RuntimeException(e);
177
- }
178
- }
179
-
180
- @Override
181
- public void add(Buffer buffer) {
182
- if (current == null) {
183
- throw new IllegalStateException(
184
- "nextFile() must be called before poll()");
185
- }
186
-
187
- try {
188
- current.write(buffer.array(), buffer.offset(), buffer.limit());
189
- } catch (IOException ex) {
190
- throw new RuntimeException(ex);
191
- } finally {
192
- buffer.release();
193
- }
194
- }
195
-
196
- @Override
197
- public void finish() {
198
- closeCurrent();
199
- }
200
-
201
- @Override
202
- public void close() {
203
- closeCurrent();
204
- }
205
-
206
- @Override
207
- public void abort() {
208
- deleteTempFile();
209
- }
210
-
211
- @Override
212
- public CommitReport commit() {
213
- CommitReport report = Exec.newCommitReport();
214
- return report;
215
- }
216
- }
217
-
218
- private void validateSequenceFormat(PluginTask task) {
219
- try {
220
- @SuppressWarnings("unused")
221
- String dontCare = String.format(Locale.ENGLISH,
222
- task.getSequenceFormat(), 0, 0);
223
- } catch (IllegalFormatException ex) {
224
- throw new ConfigException(
225
- "Invalid sequence_format: parameter for file output plugin",
226
- ex);
227
- }
228
- }
229
-
230
- @Override
231
- public ConfigDiff transaction(ConfigSource config, int taskCount,
232
- Control control) {
233
- PluginTask task = config.loadConfig(PluginTask.class);
234
-
235
- validateSequenceFormat(task);
236
-
237
- return resume(task.dump(), taskCount, control);
238
- }
239
-
240
- @Override
241
- public ConfigDiff resume(TaskSource taskSource, int taskCount,
242
- Control control) {
243
- control.run(taskSource);
244
- return Exec.newConfigDiff();
245
- }
246
-
247
- @Override
248
- public void cleanup(TaskSource taskSource, int taskCount,
249
- List<CommitReport> successCommitReports) {
250
- }
251
-
252
- @Override
253
- public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
254
- PluginTask task = taskSource.loadTask(PluginTask.class);
255
-
256
- return new S3FileOutput(task, taskIndex);
257
- }
258
- }
1
+ package org.embulk.output;
2
+
3
+ import java.io.IOException;
4
+ import java.io.OutputStream;
5
+ import java.nio.file.Files;
6
+ import java.nio.file.Path;
7
+ import java.util.IllegalFormatException;
8
+ import java.util.List;
9
+ import java.util.Locale;
10
+
11
+ import org.embulk.config.CommitReport;
12
+ import org.embulk.config.Config;
13
+ import org.embulk.config.ConfigDefault;
14
+ import org.embulk.config.ConfigDiff;
15
+ import org.embulk.config.ConfigException;
16
+ import org.embulk.config.ConfigSource;
17
+ import org.embulk.config.Task;
18
+ import org.embulk.config.TaskSource;
19
+ import org.embulk.spi.Buffer;
20
+ import org.embulk.spi.Exec;
21
+ import org.embulk.spi.FileOutput;
22
+ import org.embulk.spi.FileOutputPlugin;
23
+ import org.embulk.spi.TransactionalFileOutput;
24
+ import org.slf4j.Logger;
25
+
26
+ import com.amazonaws.ClientConfiguration;
27
+ import com.amazonaws.auth.BasicAWSCredentials;
28
+ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
29
+ import com.amazonaws.services.s3.AmazonS3Client;
30
+ import com.amazonaws.services.s3.model.PutObjectRequest;
31
+ import com.google.common.base.Optional;
32
+
33
+ public class S3FileOutputPlugin implements FileOutputPlugin {
34
+ public interface PluginTask extends Task {
35
+ @Config("path_prefix")
36
+ public String getPathPrefix();
37
+
38
+ @Config("file_ext")
39
+ public String getFileNameExtension();
40
+
41
+ @Config("sequence_format")
42
+ @ConfigDefault("\".%03d.%02d\"")
43
+ public String getSequenceFormat();
44
+
45
+ @Config("bucket")
46
+ public String getBucket();
47
+
48
+ @Config("endpoint")
49
+ public String getEndpoint();
50
+
51
+ @Config("access_key_id")
52
+ @ConfigDefault("null")
53
+ public Optional<String> getAccessKeyId();
54
+
55
+ @Config("secret_access_key")
56
+ @ConfigDefault("null")
57
+ public Optional<String> getSecretAccessKey();
58
+
59
+ @Config("tmp_path_prefix")
60
+ @ConfigDefault("\"embulk-output-s3-\"")
61
+ public String getTempPathPrefix();
62
+ }
63
+
64
+ public static class S3FileOutput implements FileOutput,
65
+ TransactionalFileOutput {
66
+ private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
67
+
68
+ private final String bucket;
69
+ private final String pathPrefix;
70
+ private final String sequenceFormat;
71
+ private final String fileNameExtension;
72
+ private final String tempPathPrefix;
73
+
74
+ private int taskIndex;
75
+ private int fileIndex;
76
+ private AmazonS3Client client;
77
+ private OutputStream current;
78
+ private Path tempFilePath;
79
+
80
+ private static AmazonS3Client newS3Client(PluginTask task) {
81
+ AmazonS3Client client = null;
82
+ try {
83
+ if (task.getAccessKeyId().isPresent()) {
84
+ BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
85
+ task.getAccessKeyId().get(), task.getSecretAccessKey().get());
86
+
87
+ ClientConfiguration config = new ClientConfiguration();
88
+ // TODO: Support more configurations.
89
+
90
+ client = new AmazonS3Client(basicAWSCredentials, config);
91
+ } else {
92
+ if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
93
+ client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
94
+ } else { // IAM ROLE
95
+ client = new AmazonS3Client();
96
+ }
97
+ }
98
+ client.setEndpoint(task.getEndpoint());
99
+ client.isRequesterPaysEnabled(task.getBucket()); // check s3 access.
100
+ } catch (Exception e) {
101
+ throw new RuntimeException("can't call S3 API. Please check your access_key_id / secret_access_key or s3_region configuration.", e);
102
+ }
103
+
104
+ return client;
105
+ }
106
+
107
+ public S3FileOutput(PluginTask task, int taskIndex) {
108
+ this.taskIndex = taskIndex;
109
+ this.client = newS3Client(task);
110
+ this.bucket = task.getBucket();
111
+ this.pathPrefix = task.getPathPrefix();
112
+ this.sequenceFormat = task.getSequenceFormat();
113
+ this.fileNameExtension = task.getFileNameExtension();
114
+ this.tempPathPrefix = task.getTempPathPrefix();
115
+ }
116
+
117
+ private static Path newTempFile(String prefix) throws IOException {
118
+ return Files.createTempFile(prefix, null);
119
+ }
120
+
121
+ private void deleteTempFile() {
122
+ if (tempFilePath == null) {
123
+ return;
124
+ }
125
+
126
+ try {
127
+ Files.delete(tempFilePath);
128
+ tempFilePath = null;
129
+ } catch (IOException e) {
130
+ throw new RuntimeException(e);
131
+ }
132
+ }
133
+
134
+ private String buildCurrentKey() {
135
+ String sequence = String.format(sequenceFormat, taskIndex,
136
+ fileIndex);
137
+ return pathPrefix + sequence + fileNameExtension;
138
+ }
139
+
140
+ private void putFile(Path from, String key) {
141
+ PutObjectRequest request = new PutObjectRequest(bucket, key,
142
+ from.toFile());
143
+ client.putObject(request);
144
+ }
145
+
146
+ private void closeCurrent() {
147
+ if (current == null) {
148
+ return;
149
+ }
150
+
151
+ try {
152
+ putFile(tempFilePath, buildCurrentKey());
153
+ fileIndex++;
154
+ } finally {
155
+ try {
156
+ current.close();
157
+ current = null;
158
+ } catch (IOException e) {
159
+ throw new RuntimeException(e);
160
+ } finally {
161
+ deleteTempFile();
162
+ }
163
+ }
164
+ }
165
+
166
+ @Override
167
+ public void nextFile() {
168
+ closeCurrent();
169
+
170
+ try {
171
+ tempFilePath = newTempFile(tempPathPrefix);
172
+
173
+ log.info("Writing S3 file '{}'", buildCurrentKey());
174
+
175
+ current = Files.newOutputStream(tempFilePath);
176
+ } catch (IOException e) {
177
+ throw new RuntimeException(e);
178
+ }
179
+ }
180
+
181
+ @Override
182
+ public void add(Buffer buffer) {
183
+ if (current == null) {
184
+ throw new IllegalStateException(
185
+ "nextFile() must be called before poll()");
186
+ }
187
+
188
+ try {
189
+ current.write(buffer.array(), buffer.offset(), buffer.limit());
190
+ } catch (IOException ex) {
191
+ throw new RuntimeException(ex);
192
+ } finally {
193
+ buffer.release();
194
+ }
195
+ }
196
+
197
+ @Override
198
+ public void finish() {
199
+ closeCurrent();
200
+ }
201
+
202
+ @Override
203
+ public void close() {
204
+ closeCurrent();
205
+ }
206
+
207
+ @Override
208
+ public void abort() {
209
+ deleteTempFile();
210
+ }
211
+
212
+ @Override
213
+ public CommitReport commit() {
214
+ CommitReport report = Exec.newCommitReport();
215
+ return report;
216
+ }
217
+ }
218
+
219
+ private void validateSequenceFormat(PluginTask task) {
220
+ try {
221
+ @SuppressWarnings("unused")
222
+ String dontCare = String.format(Locale.ENGLISH,
223
+ task.getSequenceFormat(), 0, 0);
224
+ } catch (IllegalFormatException ex) {
225
+ throw new ConfigException(
226
+ "Invalid sequence_format: parameter for file output plugin",
227
+ ex);
228
+ }
229
+ }
230
+
231
+ @Override
232
+ public ConfigDiff transaction(ConfigSource config, int taskCount,
233
+ Control control) {
234
+ PluginTask task = config.loadConfig(PluginTask.class);
235
+
236
+ validateSequenceFormat(task);
237
+
238
+ return resume(task.dump(), taskCount, control);
239
+ }
240
+
241
+ @Override
242
+ public ConfigDiff resume(TaskSource taskSource, int taskCount,
243
+ Control control) {
244
+ control.run(taskSource);
245
+ return Exec.newConfigDiff();
246
+ }
247
+
248
+ @Override
249
+ public void cleanup(TaskSource taskSource, int taskCount,
250
+ List<CommitReport> successCommitReports) {
251
+ }
252
+
253
+ @Override
254
+ public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
255
+ PluginTask task = taskSource.loadTask(PluginTask.class);
256
+
257
+ return new S3FileOutput(task, taskIndex);
258
+ }
259
+ }
@@ -1,5 +1,5 @@
1
- package org.embulk.output;
2
-
3
- public class TestS3FileOutputPlugin
4
- {
5
- }
1
+ package org.embulk.output;
2
+
3
+ public class TestS3FileOutputPlugin
4
+ {
5
+ }
metadata CHANGED
@@ -1,44 +1,44 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Takayama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-15 00:00:00.000000000 Z
11
+ date: 2015-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - ~>
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - ~>
32
25
  - !ruby/object:Gem::Version
33
- version: '10.0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: '10.0'
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
41
- description: '"Stores files on S3."'
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ description: Stores files on S3.
42
42
  email:
43
43
  - learn.libra@gmail.com
44
44
  executables: []
@@ -61,9 +61,10 @@ files:
61
61
  - classpath/aws-java-sdk-s3-1.9.24.jar
62
62
  - classpath/commons-codec-1.6.jar
63
63
  - classpath/commons-logging-1.1.3.jar
64
- - classpath/embulk-output-s3-0.1.0.jar
64
+ - classpath/embulk-output-s3-0.2.0.jar
65
65
  - classpath/httpclient-4.3.4.jar
66
66
  - classpath/httpcore-4.3.2.jar
67
+ - classpath/joda-time-2.8.2.jar
67
68
  homepage: https://github.com/llibra/embulk-output-s3
68
69
  licenses:
69
70
  - MIT
Binary file