embulk-output-s3 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ec5fe08d2ad38fad94f03679479e3653cfac20cb
4
- data.tar.gz: 6d363a602b57f9b9315b68a16602ca6a0af294c6
3
+ metadata.gz: 9da3670f4971d655c13d7a81cd908cc510a86b6f
4
+ data.tar.gz: 7e0bb6615000fd7a091057ccb33b385d3773e729
5
5
  SHA512:
6
- metadata.gz: bd0d1aa10571f12e130040a5073a5b7fcc649576f81f5bb83f8f32e12aba031fa56061a8c78d1308cf52445921e50754ab4490ffae2cdfa0dd49a3990003247e
7
- data.tar.gz: df38d944942edb611ff64ac85a243246a61c31ab03350baa811fd3f91fef77cc3f77b3024860f8ed9c73b4415b12c4aeb8ccf345e93f05db316cf8021de0e6f6
6
+ metadata.gz: 7cc3cf1287546ba8a40d69114146146ae1031ee5786789f072a3e80b6e1a682d0eabfe6504ac2abde1ac59deef0b557ccafe74899db8efb4dad6ccb1a1522028
7
+ data.tar.gz: 46e4f2a4a1a5b958f09e93e9ffc7299bfa6b0afbe6c5d7293b56fcb6083fb4b2a86019dcd15c29a27db3fbce9a9b105051f1f4ed2fb8a79702f2302672e5143b
data/.gitignore CHANGED
@@ -1,11 +1,11 @@
1
- *~
2
- /pkg/
3
- /tmp/
4
- .gradle/
5
- /classpath/
6
- build/
7
- .idea
8
- .settings/
9
- bin/
10
- .classpath
11
- .project
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ .gradle/
5
+ /classpath/
6
+ build/
7
+ .idea
8
+ .settings/
9
+ bin/
10
+ .classpath
11
+ .project
data/README.md CHANGED
@@ -1,39 +1,39 @@
1
- # S3 file output plugin for Embulk
2
-
3
- ## Overview
4
-
5
- * **Plugin type**: file output
6
- * **Load all or nothing**: no
7
- * **Resume supported**: yes
8
- * **Cleanup supported**: yes
9
-
10
- ## Configuration
11
-
12
- - **path_prefix**: prefix of target keys (string, required)
13
- - **file_ext**: suffix of target keys (string, required)
14
- - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
- - **bucket**: S3 bucket name (string, required)
16
- - **endpoint**: S3 endpoint login user name (string, optional)
17
- - **access_key_id**: AWS access key id (string, required)
18
- - **secret_access_key**: AWS secret key (string, required)
19
- - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
-
21
- ## Example
22
-
23
- ```yaml
24
- path_prefix: logs/out
25
- file_ext: .csv
26
- bucket: my-s3-bucket
27
- endpoint: s3-us-west-1.amazonaws.com
28
- access_key_id: ABCXYZ123ABCXYZ123
29
- secret_access_key: AbCxYz123aBcXyZ123
30
- formatter:
31
- type: csv
32
- ```
33
-
34
-
35
- ## Build
36
-
37
- ```
38
- $ ./gradlew gem
39
- ```
1
+ # S3 file output plugin for Embulk
2
+
3
+ ## Overview
4
+
5
+ * **Plugin type**: file output
6
+ * **Load all or nothing**: no
7
+ * **Resume supported**: yes
8
+ * **Cleanup supported**: yes
9
+
10
+ ## Configuration
11
+
12
+ - **path_prefix**: prefix of target keys (string, required)
13
+ - **file_ext**: suffix of target keys (string, required)
14
+ - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
+ - **bucket**: S3 bucket name (string, required)
16
+ - **endpoint**: S3 endpoint login user name (string, optional)
17
+ - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
18
+ - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
19
+ - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
+
21
+ ## Example
22
+
23
+ ```yaml
24
+ path_prefix: logs/out
25
+ file_ext: .csv
26
+ bucket: my-s3-bucket
27
+ endpoint: s3-us-west-1.amazonaws.com
28
+ access_key_id: ABCXYZ123ABCXYZ123
29
+ secret_access_key: AbCxYz123aBcXyZ123
30
+ formatter:
31
+ type: csv
32
+ ```
33
+
34
+
35
+ ## Build
36
+
37
+ ```
38
+ $ ./gradlew gem
39
+ ```
data/build.gradle CHANGED
@@ -1,56 +1,56 @@
1
- plugins {
2
- id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
4
- id "java"
5
- }
6
- import com.github.jrubygradle.JRubyExec
7
- repositories {
8
- mavenCentral()
9
- jcenter()
10
- }
11
- configurations {
12
- provided
13
- }
14
-
15
- version = "0.1.0"
16
-
17
- dependencies {
18
- compile "org.embulk:embulk-core:0.5.2"
19
- provided "org.embulk:embulk-core:0.5.2"
20
- compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
21
- testCompile "junit:junit:4.+"
22
- }
23
-
24
- task classpath(type: Copy, dependsOn: ["jar"]) {
25
- doFirst { file("classpath").deleteDir() }
26
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
27
- into "classpath"
28
- }
29
- clean { delete 'classpath' }
30
-
31
- task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
32
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
33
- script "build/gemspec"
34
- doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
35
- }
36
-
37
- task gemspec << { file("build/gemspec").write($/
38
- Gem::Specification.new do |spec|
39
- spec.name = "${project.name}"
40
- spec.version = "${project.version}"
41
- spec.authors = ["Manabu Takayama"]
42
- spec.summary = %[S3 file output plugin for Embulk]
43
- spec.description = %["Stores files on S3."]
44
- spec.email = ["learn.libra@gmail.com"]
45
- spec.licenses = ["MIT"]
46
- spec.homepage = "https://github.com/llibra/embulk-output-s3"
47
-
48
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
49
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
50
- spec.require_paths = ["lib"]
51
-
52
- spec.add_development_dependency 'bundler', ['~> 1.0']
53
- spec.add_development_dependency 'rake', ['>= 10.0']
54
- end
55
- /$)
56
- }
1
+ plugins {
2
+ id "com.jfrog.bintray" version "1.1"
3
+ id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "java"
5
+ }
6
+ import com.github.jrubygradle.JRubyExec
7
+ repositories {
8
+ mavenCentral()
9
+ jcenter()
10
+ }
11
+ configurations {
12
+ provided
13
+ }
14
+
15
+ version = "0.2.0"
16
+
17
+ dependencies {
18
+ compile "org.embulk:embulk-core:0.5.2"
19
+ provided "org.embulk:embulk-core:0.5.2"
20
+ compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
21
+ testCompile "junit:junit:4.+"
22
+ }
23
+
24
+ task classpath(type: Copy, dependsOn: ["jar"]) {
25
+ doFirst { file("classpath").deleteDir() }
26
+ from (configurations.runtime - configurations.provided + files(jar.archivePath))
27
+ into "classpath"
28
+ }
29
+ clean { delete 'classpath' }
30
+
31
+ task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
32
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
33
+ script "build/gemspec"
34
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
35
+ }
36
+
37
+ task gemspec << { file("build/gemspec").write($/
38
+ Gem::Specification.new do |spec|
39
+ spec.name = "${project.name}"
40
+ spec.version = "${project.version}"
41
+ spec.authors = ["Manabu Takayama"]
42
+ spec.summary = %[S3 file output plugin for Embulk]
43
+ spec.description = %[Stores files on S3.]
44
+ spec.email = ["learn.libra@gmail.com"]
45
+ spec.licenses = ["MIT"]
46
+ spec.homepage = "https://github.com/llibra/embulk-output-s3"
47
+
48
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
49
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
50
+ spec.require_paths = ["lib"]
51
+
52
+ spec.add_development_dependency 'bundler', ['~> 1.0']
53
+ spec.add_development_dependency 'rake', ['>= 10.0']
54
+ end
55
+ /$)
56
+ }
Binary file
@@ -1,258 +1,259 @@
1
- package org.embulk.output;
2
-
3
- import java.io.IOException;
4
- import java.io.OutputStream;
5
- import java.nio.file.Files;
6
- import java.nio.file.Path;
7
- import java.util.IllegalFormatException;
8
- import java.util.List;
9
- import java.util.Locale;
10
-
11
- import org.embulk.config.CommitReport;
12
- import org.embulk.config.Config;
13
- import org.embulk.config.ConfigDefault;
14
- import org.embulk.config.ConfigDiff;
15
- import org.embulk.config.ConfigException;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskSource;
19
- import org.embulk.spi.Buffer;
20
- import org.embulk.spi.Exec;
21
- import org.embulk.spi.FileOutput;
22
- import org.embulk.spi.FileOutputPlugin;
23
- import org.embulk.spi.TransactionalFileOutput;
24
- import org.slf4j.Logger;
25
-
26
- import com.amazonaws.ClientConfiguration;
27
- import com.amazonaws.auth.AWSCredentials;
28
- import com.amazonaws.auth.AWSCredentialsProvider;
29
- import com.amazonaws.auth.BasicAWSCredentials;
30
- import com.amazonaws.services.s3.AmazonS3Client;
31
- import com.amazonaws.services.s3.model.PutObjectRequest;
32
-
33
- public class S3FileOutputPlugin implements FileOutputPlugin {
34
- public interface PluginTask extends Task {
35
- @Config("path_prefix")
36
- public String getPathPrefix();
37
-
38
- @Config("file_ext")
39
- public String getFileNameExtension();
40
-
41
- @Config("sequence_format")
42
- @ConfigDefault("\".%03d.%02d\"")
43
- public String getSequenceFormat();
44
-
45
- @Config("bucket")
46
- public String getBucket();
47
-
48
- @Config("endpoint")
49
- public String getEndpoint();
50
-
51
- @Config("access_key_id")
52
- public String getAccessKeyId();
53
-
54
- @Config("secret_access_key")
55
- public String getSecretAccessKey();
56
-
57
- @Config("tmp_path_prefix")
58
- @ConfigDefault("\"embulk-output-s3-\"")
59
- public String getTempPathPrefix();
60
- }
61
-
62
- public static class S3FileOutput implements FileOutput,
63
- TransactionalFileOutput {
64
- private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
65
-
66
- private final String bucket;
67
- private final String pathPrefix;
68
- private final String sequenceFormat;
69
- private final String fileNameExtension;
70
- private final String tempPathPrefix;
71
-
72
- private int taskIndex;
73
- private int fileIndex;
74
- private AmazonS3Client client;
75
- private OutputStream current;
76
- private Path tempFilePath;
77
-
78
- public static AWSCredentialsProvider getCredentialsProvider(
79
- PluginTask task) {
80
- final AWSCredentials cred = new BasicAWSCredentials(
81
- task.getAccessKeyId(), task.getSecretAccessKey());
82
- return new AWSCredentialsProvider() {
83
- @Override
84
- public AWSCredentials getCredentials() {
85
- return cred;
86
- }
87
-
88
- @Override
89
- public void refresh() {
90
- }
91
- };
92
- }
93
-
94
- private static AmazonS3Client newS3Client(PluginTask task) {
95
- AWSCredentialsProvider credentials = getCredentialsProvider(task);
96
-
97
- ClientConfiguration config = new ClientConfiguration();
98
- // TODO: Support more configurations.
99
-
100
- AmazonS3Client client = new AmazonS3Client(credentials, config);
101
- client.setEndpoint(task.getEndpoint());
102
-
103
- return client;
104
- }
105
-
106
- public S3FileOutput(PluginTask task, int taskIndex) {
107
- this.taskIndex = taskIndex;
108
- this.client = newS3Client(task);
109
- this.bucket = task.getBucket();
110
- this.pathPrefix = task.getPathPrefix();
111
- this.sequenceFormat = task.getSequenceFormat();
112
- this.fileNameExtension = task.getFileNameExtension();
113
- this.tempPathPrefix = task.getTempPathPrefix();
114
- }
115
-
116
- private static Path newTempFile(String prefix) throws IOException {
117
- return Files.createTempFile(prefix, null);
118
- }
119
-
120
- private void deleteTempFile() {
121
- if (tempFilePath == null) {
122
- return;
123
- }
124
-
125
- try {
126
- Files.delete(tempFilePath);
127
- tempFilePath = null;
128
- } catch (IOException e) {
129
- throw new RuntimeException(e);
130
- }
131
- }
132
-
133
- private String buildCurrentKey() {
134
- String sequence = String.format(sequenceFormat, taskIndex,
135
- fileIndex);
136
- return pathPrefix + sequence + fileNameExtension;
137
- }
138
-
139
- private void putFile(Path from, String key) {
140
- PutObjectRequest request = new PutObjectRequest(bucket, key,
141
- from.toFile());
142
- client.putObject(request);
143
- }
144
-
145
- private void closeCurrent() {
146
- if (current == null) {
147
- return;
148
- }
149
-
150
- try {
151
- putFile(tempFilePath, buildCurrentKey());
152
- fileIndex++;
153
- } finally {
154
- try {
155
- current.close();
156
- current = null;
157
- } catch (IOException e) {
158
- throw new RuntimeException(e);
159
- } finally {
160
- deleteTempFile();
161
- }
162
- }
163
- }
164
-
165
- @Override
166
- public void nextFile() {
167
- closeCurrent();
168
-
169
- try {
170
- tempFilePath = newTempFile(tempPathPrefix);
171
-
172
- log.info("Writing S3 file '{}'", buildCurrentKey());
173
-
174
- current = Files.newOutputStream(tempFilePath);
175
- } catch (IOException e) {
176
- throw new RuntimeException(e);
177
- }
178
- }
179
-
180
- @Override
181
- public void add(Buffer buffer) {
182
- if (current == null) {
183
- throw new IllegalStateException(
184
- "nextFile() must be called before poll()");
185
- }
186
-
187
- try {
188
- current.write(buffer.array(), buffer.offset(), buffer.limit());
189
- } catch (IOException ex) {
190
- throw new RuntimeException(ex);
191
- } finally {
192
- buffer.release();
193
- }
194
- }
195
-
196
- @Override
197
- public void finish() {
198
- closeCurrent();
199
- }
200
-
201
- @Override
202
- public void close() {
203
- closeCurrent();
204
- }
205
-
206
- @Override
207
- public void abort() {
208
- deleteTempFile();
209
- }
210
-
211
- @Override
212
- public CommitReport commit() {
213
- CommitReport report = Exec.newCommitReport();
214
- return report;
215
- }
216
- }
217
-
218
- private void validateSequenceFormat(PluginTask task) {
219
- try {
220
- @SuppressWarnings("unused")
221
- String dontCare = String.format(Locale.ENGLISH,
222
- task.getSequenceFormat(), 0, 0);
223
- } catch (IllegalFormatException ex) {
224
- throw new ConfigException(
225
- "Invalid sequence_format: parameter for file output plugin",
226
- ex);
227
- }
228
- }
229
-
230
- @Override
231
- public ConfigDiff transaction(ConfigSource config, int taskCount,
232
- Control control) {
233
- PluginTask task = config.loadConfig(PluginTask.class);
234
-
235
- validateSequenceFormat(task);
236
-
237
- return resume(task.dump(), taskCount, control);
238
- }
239
-
240
- @Override
241
- public ConfigDiff resume(TaskSource taskSource, int taskCount,
242
- Control control) {
243
- control.run(taskSource);
244
- return Exec.newConfigDiff();
245
- }
246
-
247
- @Override
248
- public void cleanup(TaskSource taskSource, int taskCount,
249
- List<CommitReport> successCommitReports) {
250
- }
251
-
252
- @Override
253
- public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
254
- PluginTask task = taskSource.loadTask(PluginTask.class);
255
-
256
- return new S3FileOutput(task, taskIndex);
257
- }
258
- }
1
+ package org.embulk.output;
2
+
3
+ import java.io.IOException;
4
+ import java.io.OutputStream;
5
+ import java.nio.file.Files;
6
+ import java.nio.file.Path;
7
+ import java.util.IllegalFormatException;
8
+ import java.util.List;
9
+ import java.util.Locale;
10
+
11
+ import org.embulk.config.CommitReport;
12
+ import org.embulk.config.Config;
13
+ import org.embulk.config.ConfigDefault;
14
+ import org.embulk.config.ConfigDiff;
15
+ import org.embulk.config.ConfigException;
16
+ import org.embulk.config.ConfigSource;
17
+ import org.embulk.config.Task;
18
+ import org.embulk.config.TaskSource;
19
+ import org.embulk.spi.Buffer;
20
+ import org.embulk.spi.Exec;
21
+ import org.embulk.spi.FileOutput;
22
+ import org.embulk.spi.FileOutputPlugin;
23
+ import org.embulk.spi.TransactionalFileOutput;
24
+ import org.slf4j.Logger;
25
+
26
+ import com.amazonaws.ClientConfiguration;
27
+ import com.amazonaws.auth.BasicAWSCredentials;
28
+ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
29
+ import com.amazonaws.services.s3.AmazonS3Client;
30
+ import com.amazonaws.services.s3.model.PutObjectRequest;
31
+ import com.google.common.base.Optional;
32
+
33
+ public class S3FileOutputPlugin implements FileOutputPlugin {
34
+ public interface PluginTask extends Task {
35
+ @Config("path_prefix")
36
+ public String getPathPrefix();
37
+
38
+ @Config("file_ext")
39
+ public String getFileNameExtension();
40
+
41
+ @Config("sequence_format")
42
+ @ConfigDefault("\".%03d.%02d\"")
43
+ public String getSequenceFormat();
44
+
45
+ @Config("bucket")
46
+ public String getBucket();
47
+
48
+ @Config("endpoint")
49
+ public String getEndpoint();
50
+
51
+ @Config("access_key_id")
52
+ @ConfigDefault("null")
53
+ public Optional<String> getAccessKeyId();
54
+
55
+ @Config("secret_access_key")
56
+ @ConfigDefault("null")
57
+ public Optional<String> getSecretAccessKey();
58
+
59
+ @Config("tmp_path_prefix")
60
+ @ConfigDefault("\"embulk-output-s3-\"")
61
+ public String getTempPathPrefix();
62
+ }
63
+
64
+ public static class S3FileOutput implements FileOutput,
65
+ TransactionalFileOutput {
66
+ private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
67
+
68
+ private final String bucket;
69
+ private final String pathPrefix;
70
+ private final String sequenceFormat;
71
+ private final String fileNameExtension;
72
+ private final String tempPathPrefix;
73
+
74
+ private int taskIndex;
75
+ private int fileIndex;
76
+ private AmazonS3Client client;
77
+ private OutputStream current;
78
+ private Path tempFilePath;
79
+
80
+ private static AmazonS3Client newS3Client(PluginTask task) {
81
+ AmazonS3Client client = null;
82
+ try {
83
+ if (task.getAccessKeyId().isPresent()) {
84
+ BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
85
+ task.getAccessKeyId().get(), task.getSecretAccessKey().get());
86
+
87
+ ClientConfiguration config = new ClientConfiguration();
88
+ // TODO: Support more configurations.
89
+
90
+ client = new AmazonS3Client(basicAWSCredentials, config);
91
+ } else {
92
+ if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
93
+ client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
94
+ } else { // IAM ROLE
95
+ client = new AmazonS3Client();
96
+ }
97
+ }
98
+ client.setEndpoint(task.getEndpoint());
99
+ client.isRequesterPaysEnabled(task.getBucket()); // check s3 access.
100
+ } catch (Exception e) {
101
+ throw new RuntimeException("can't call S3 API. Please check your access_key_id / secret_access_key or s3_region configuration.", e);
102
+ }
103
+
104
+ return client;
105
+ }
106
+
107
+ public S3FileOutput(PluginTask task, int taskIndex) {
108
+ this.taskIndex = taskIndex;
109
+ this.client = newS3Client(task);
110
+ this.bucket = task.getBucket();
111
+ this.pathPrefix = task.getPathPrefix();
112
+ this.sequenceFormat = task.getSequenceFormat();
113
+ this.fileNameExtension = task.getFileNameExtension();
114
+ this.tempPathPrefix = task.getTempPathPrefix();
115
+ }
116
+
117
+ private static Path newTempFile(String prefix) throws IOException {
118
+ return Files.createTempFile(prefix, null);
119
+ }
120
+
121
+ private void deleteTempFile() {
122
+ if (tempFilePath == null) {
123
+ return;
124
+ }
125
+
126
+ try {
127
+ Files.delete(tempFilePath);
128
+ tempFilePath = null;
129
+ } catch (IOException e) {
130
+ throw new RuntimeException(e);
131
+ }
132
+ }
133
+
134
+ private String buildCurrentKey() {
135
+ String sequence = String.format(sequenceFormat, taskIndex,
136
+ fileIndex);
137
+ return pathPrefix + sequence + fileNameExtension;
138
+ }
139
+
140
+ private void putFile(Path from, String key) {
141
+ PutObjectRequest request = new PutObjectRequest(bucket, key,
142
+ from.toFile());
143
+ client.putObject(request);
144
+ }
145
+
146
+ private void closeCurrent() {
147
+ if (current == null) {
148
+ return;
149
+ }
150
+
151
+ try {
152
+ putFile(tempFilePath, buildCurrentKey());
153
+ fileIndex++;
154
+ } finally {
155
+ try {
156
+ current.close();
157
+ current = null;
158
+ } catch (IOException e) {
159
+ throw new RuntimeException(e);
160
+ } finally {
161
+ deleteTempFile();
162
+ }
163
+ }
164
+ }
165
+
166
+ @Override
167
+ public void nextFile() {
168
+ closeCurrent();
169
+
170
+ try {
171
+ tempFilePath = newTempFile(tempPathPrefix);
172
+
173
+ log.info("Writing S3 file '{}'", buildCurrentKey());
174
+
175
+ current = Files.newOutputStream(tempFilePath);
176
+ } catch (IOException e) {
177
+ throw new RuntimeException(e);
178
+ }
179
+ }
180
+
181
+ @Override
182
+ public void add(Buffer buffer) {
183
+ if (current == null) {
184
+ throw new IllegalStateException(
185
+ "nextFile() must be called before poll()");
186
+ }
187
+
188
+ try {
189
+ current.write(buffer.array(), buffer.offset(), buffer.limit());
190
+ } catch (IOException ex) {
191
+ throw new RuntimeException(ex);
192
+ } finally {
193
+ buffer.release();
194
+ }
195
+ }
196
+
197
+ @Override
198
+ public void finish() {
199
+ closeCurrent();
200
+ }
201
+
202
+ @Override
203
+ public void close() {
204
+ closeCurrent();
205
+ }
206
+
207
+ @Override
208
+ public void abort() {
209
+ deleteTempFile();
210
+ }
211
+
212
+ @Override
213
+ public CommitReport commit() {
214
+ CommitReport report = Exec.newCommitReport();
215
+ return report;
216
+ }
217
+ }
218
+
219
+ private void validateSequenceFormat(PluginTask task) {
220
+ try {
221
+ @SuppressWarnings("unused")
222
+ String dontCare = String.format(Locale.ENGLISH,
223
+ task.getSequenceFormat(), 0, 0);
224
+ } catch (IllegalFormatException ex) {
225
+ throw new ConfigException(
226
+ "Invalid sequence_format: parameter for file output plugin",
227
+ ex);
228
+ }
229
+ }
230
+
231
+ @Override
232
+ public ConfigDiff transaction(ConfigSource config, int taskCount,
233
+ Control control) {
234
+ PluginTask task = config.loadConfig(PluginTask.class);
235
+
236
+ validateSequenceFormat(task);
237
+
238
+ return resume(task.dump(), taskCount, control);
239
+ }
240
+
241
+ @Override
242
+ public ConfigDiff resume(TaskSource taskSource, int taskCount,
243
+ Control control) {
244
+ control.run(taskSource);
245
+ return Exec.newConfigDiff();
246
+ }
247
+
248
+ @Override
249
+ public void cleanup(TaskSource taskSource, int taskCount,
250
+ List<CommitReport> successCommitReports) {
251
+ }
252
+
253
+ @Override
254
+ public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
255
+ PluginTask task = taskSource.loadTask(PluginTask.class);
256
+
257
+ return new S3FileOutput(task, taskIndex);
258
+ }
259
+ }
@@ -1,5 +1,5 @@
1
- package org.embulk.output;
2
-
3
- public class TestS3FileOutputPlugin
4
- {
5
- }
1
+ package org.embulk.output;
2
+
3
+ public class TestS3FileOutputPlugin
4
+ {
5
+ }
metadata CHANGED
@@ -1,44 +1,44 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Takayama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-03-15 00:00:00.000000000 Z
11
+ date: 2015-08-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- name: bundler
15
- version_requirements: !ruby/object:Gem::Requirement
16
- requirements:
17
- - - ~>
18
- - !ruby/object:Gem::Version
19
- version: '1.0'
20
14
  requirement: !ruby/object:Gem::Requirement
21
15
  requirements:
22
16
  - - ~>
23
17
  - !ruby/object:Gem::Version
24
18
  version: '1.0'
19
+ name: bundler
25
20
  prerelease: false
26
21
  type: :development
27
- - !ruby/object:Gem::Dependency
28
- name: rake
29
22
  version_requirements: !ruby/object:Gem::Requirement
30
23
  requirements:
31
- - - '>='
24
+ - - ~>
32
25
  - !ruby/object:Gem::Version
33
- version: '10.0'
26
+ version: '1.0'
27
+ - !ruby/object:Gem::Dependency
34
28
  requirement: !ruby/object:Gem::Requirement
35
29
  requirements:
36
30
  - - '>='
37
31
  - !ruby/object:Gem::Version
38
32
  version: '10.0'
33
+ name: rake
39
34
  prerelease: false
40
35
  type: :development
41
- description: '"Stores files on S3."'
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '>='
39
+ - !ruby/object:Gem::Version
40
+ version: '10.0'
41
+ description: Stores files on S3.
42
42
  email:
43
43
  - learn.libra@gmail.com
44
44
  executables: []
@@ -61,9 +61,10 @@ files:
61
61
  - classpath/aws-java-sdk-s3-1.9.24.jar
62
62
  - classpath/commons-codec-1.6.jar
63
63
  - classpath/commons-logging-1.1.3.jar
64
- - classpath/embulk-output-s3-0.1.0.jar
64
+ - classpath/embulk-output-s3-0.2.0.jar
65
65
  - classpath/httpclient-4.3.4.jar
66
66
  - classpath/httpcore-4.3.2.jar
67
+ - classpath/joda-time-2.8.2.jar
67
68
  homepage: https://github.com/llibra/embulk-output-s3
68
69
  licenses:
69
70
  - MIT
Binary file