embulk-output-s3 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +11 -11
- data/README.md +39 -39
- data/build.gradle +56 -56
- data/classpath/embulk-output-s3-0.2.0.jar +0 -0
- data/classpath/joda-time-2.8.2.jar +0 -0
- data/src/main/java/org/embulk/output/S3FileOutputPlugin.java +259 -258
- data/src/test/java/org/embulk/output/TestS3FileOutputPlugin.java +5 -5
- metadata +15 -14
- data/classpath/embulk-output-s3-0.1.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9da3670f4971d655c13d7a81cd908cc510a86b6f
|
4
|
+
data.tar.gz: 7e0bb6615000fd7a091057ccb33b385d3773e729
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7cc3cf1287546ba8a40d69114146146ae1031ee5786789f072a3e80b6e1a682d0eabfe6504ac2abde1ac59deef0b557ccafe74899db8efb4dad6ccb1a1522028
|
7
|
+
data.tar.gz: 46e4f2a4a1a5b958f09e93e9ffc7299bfa6b0afbe6c5d7293b56fcb6083fb4b2a86019dcd15c29a27db3fbce9a9b105051f1f4ed2fb8a79702f2302672e5143b
|
data/.gitignore
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
*~
|
2
|
-
/pkg/
|
3
|
-
/tmp/
|
4
|
-
.gradle/
|
5
|
-
/classpath/
|
6
|
-
build/
|
7
|
-
.idea
|
8
|
-
.settings/
|
9
|
-
bin/
|
10
|
-
.classpath
|
11
|
-
.project
|
1
|
+
*~
|
2
|
+
/pkg/
|
3
|
+
/tmp/
|
4
|
+
.gradle/
|
5
|
+
/classpath/
|
6
|
+
build/
|
7
|
+
.idea
|
8
|
+
.settings/
|
9
|
+
bin/
|
10
|
+
.classpath
|
11
|
+
.project
|
data/README.md
CHANGED
@@ -1,39 +1,39 @@
|
|
1
|
-
# S3 file output plugin for Embulk
|
2
|
-
|
3
|
-
## Overview
|
4
|
-
|
5
|
-
* **Plugin type**: file output
|
6
|
-
* **Load all or nothing**: no
|
7
|
-
* **Resume supported**: yes
|
8
|
-
* **Cleanup supported**: yes
|
9
|
-
|
10
|
-
## Configuration
|
11
|
-
|
12
|
-
- **path_prefix**: prefix of target keys (string, required)
|
13
|
-
- **file_ext**: suffix of target keys (string, required)
|
14
|
-
- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
|
15
|
-
- **bucket**: S3 bucket name (string, required)
|
16
|
-
- **endpoint**: S3 endpoint login user name (string, optional)
|
17
|
-
- **access_key_id**: AWS access key id (string,
|
18
|
-
- **secret_access_key**: AWS secret key (string,
|
19
|
-
- **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
|
20
|
-
|
21
|
-
## Example
|
22
|
-
|
23
|
-
```yaml
|
24
|
-
path_prefix: logs/out
|
25
|
-
file_ext: .csv
|
26
|
-
bucket: my-s3-bucket
|
27
|
-
endpoint: s3-us-west-1.amazonaws.com
|
28
|
-
access_key_id: ABCXYZ123ABCXYZ123
|
29
|
-
secret_access_key: AbCxYz123aBcXyZ123
|
30
|
-
formatter:
|
31
|
-
type: csv
|
32
|
-
```
|
33
|
-
|
34
|
-
|
35
|
-
## Build
|
36
|
-
|
37
|
-
```
|
38
|
-
$ ./gradlew gem
|
39
|
-
```
|
1
|
+
# S3 file output plugin for Embulk
|
2
|
+
|
3
|
+
## Overview
|
4
|
+
|
5
|
+
* **Plugin type**: file output
|
6
|
+
* **Load all or nothing**: no
|
7
|
+
* **Resume supported**: yes
|
8
|
+
* **Cleanup supported**: yes
|
9
|
+
|
10
|
+
## Configuration
|
11
|
+
|
12
|
+
- **path_prefix**: prefix of target keys (string, required)
|
13
|
+
- **file_ext**: suffix of target keys (string, required)
|
14
|
+
- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
|
15
|
+
- **bucket**: S3 bucket name (string, required)
|
16
|
+
- **endpoint**: S3 endpoint login user name (string, optional)
|
17
|
+
- **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
18
|
+
- **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
19
|
+
- **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
|
20
|
+
|
21
|
+
## Example
|
22
|
+
|
23
|
+
```yaml
|
24
|
+
path_prefix: logs/out
|
25
|
+
file_ext: .csv
|
26
|
+
bucket: my-s3-bucket
|
27
|
+
endpoint: s3-us-west-1.amazonaws.com
|
28
|
+
access_key_id: ABCXYZ123ABCXYZ123
|
29
|
+
secret_access_key: AbCxYz123aBcXyZ123
|
30
|
+
formatter:
|
31
|
+
type: csv
|
32
|
+
```
|
33
|
+
|
34
|
+
|
35
|
+
## Build
|
36
|
+
|
37
|
+
```
|
38
|
+
$ ./gradlew gem
|
39
|
+
```
|
data/build.gradle
CHANGED
@@ -1,56 +1,56 @@
|
|
1
|
-
plugins {
|
2
|
-
id "com.jfrog.bintray" version "1.1"
|
3
|
-
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
-
id "java"
|
5
|
-
}
|
6
|
-
import com.github.jrubygradle.JRubyExec
|
7
|
-
repositories {
|
8
|
-
mavenCentral()
|
9
|
-
jcenter()
|
10
|
-
}
|
11
|
-
configurations {
|
12
|
-
provided
|
13
|
-
}
|
14
|
-
|
15
|
-
version = "0.
|
16
|
-
|
17
|
-
dependencies {
|
18
|
-
compile "org.embulk:embulk-core:0.5.2"
|
19
|
-
provided "org.embulk:embulk-core:0.5.2"
|
20
|
-
compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
|
21
|
-
testCompile "junit:junit:4.+"
|
22
|
-
}
|
23
|
-
|
24
|
-
task classpath(type: Copy, dependsOn: ["jar"]) {
|
25
|
-
doFirst { file("classpath").deleteDir() }
|
26
|
-
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
27
|
-
into "classpath"
|
28
|
-
}
|
29
|
-
clean { delete 'classpath' }
|
30
|
-
|
31
|
-
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
32
|
-
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
33
|
-
script "build/gemspec"
|
34
|
-
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
35
|
-
}
|
36
|
-
|
37
|
-
task gemspec << { file("build/gemspec").write($/
|
38
|
-
Gem::Specification.new do |spec|
|
39
|
-
spec.name = "${project.name}"
|
40
|
-
spec.version = "${project.version}"
|
41
|
-
spec.authors = ["Manabu Takayama"]
|
42
|
-
spec.summary = %[S3 file output plugin for Embulk]
|
43
|
-
spec.description = %[
|
44
|
-
spec.email = ["learn.libra@gmail.com"]
|
45
|
-
spec.licenses = ["MIT"]
|
46
|
-
spec.homepage = "https://github.com/llibra/embulk-output-s3"
|
47
|
-
|
48
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
49
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
50
|
-
spec.require_paths = ["lib"]
|
51
|
-
|
52
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
53
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
54
|
-
end
|
55
|
-
/$)
|
56
|
-
}
|
1
|
+
plugins {
|
2
|
+
id "com.jfrog.bintray" version "1.1"
|
3
|
+
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "java"
|
5
|
+
}
|
6
|
+
import com.github.jrubygradle.JRubyExec
|
7
|
+
repositories {
|
8
|
+
mavenCentral()
|
9
|
+
jcenter()
|
10
|
+
}
|
11
|
+
configurations {
|
12
|
+
provided
|
13
|
+
}
|
14
|
+
|
15
|
+
version = "0.2.0"
|
16
|
+
|
17
|
+
dependencies {
|
18
|
+
compile "org.embulk:embulk-core:0.5.2"
|
19
|
+
provided "org.embulk:embulk-core:0.5.2"
|
20
|
+
compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
|
21
|
+
testCompile "junit:junit:4.+"
|
22
|
+
}
|
23
|
+
|
24
|
+
task classpath(type: Copy, dependsOn: ["jar"]) {
|
25
|
+
doFirst { file("classpath").deleteDir() }
|
26
|
+
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
27
|
+
into "classpath"
|
28
|
+
}
|
29
|
+
clean { delete 'classpath' }
|
30
|
+
|
31
|
+
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
32
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
33
|
+
script "build/gemspec"
|
34
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
35
|
+
}
|
36
|
+
|
37
|
+
task gemspec << { file("build/gemspec").write($/
|
38
|
+
Gem::Specification.new do |spec|
|
39
|
+
spec.name = "${project.name}"
|
40
|
+
spec.version = "${project.version}"
|
41
|
+
spec.authors = ["Manabu Takayama"]
|
42
|
+
spec.summary = %[S3 file output plugin for Embulk]
|
43
|
+
spec.description = %[Stores files on S3.]
|
44
|
+
spec.email = ["learn.libra@gmail.com"]
|
45
|
+
spec.licenses = ["MIT"]
|
46
|
+
spec.homepage = "https://github.com/llibra/embulk-output-s3"
|
47
|
+
|
48
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
49
|
+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
50
|
+
spec.require_paths = ["lib"]
|
51
|
+
|
52
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
53
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
54
|
+
end
|
55
|
+
/$)
|
56
|
+
}
|
Binary file
|
Binary file
|
@@ -1,258 +1,259 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import java.io.IOException;
|
4
|
-
import java.io.OutputStream;
|
5
|
-
import java.nio.file.Files;
|
6
|
-
import java.nio.file.Path;
|
7
|
-
import java.util.IllegalFormatException;
|
8
|
-
import java.util.List;
|
9
|
-
import java.util.Locale;
|
10
|
-
|
11
|
-
import org.embulk.config.CommitReport;
|
12
|
-
import org.embulk.config.Config;
|
13
|
-
import org.embulk.config.ConfigDefault;
|
14
|
-
import org.embulk.config.ConfigDiff;
|
15
|
-
import org.embulk.config.ConfigException;
|
16
|
-
import org.embulk.config.ConfigSource;
|
17
|
-
import org.embulk.config.Task;
|
18
|
-
import org.embulk.config.TaskSource;
|
19
|
-
import org.embulk.spi.Buffer;
|
20
|
-
import org.embulk.spi.Exec;
|
21
|
-
import org.embulk.spi.FileOutput;
|
22
|
-
import org.embulk.spi.FileOutputPlugin;
|
23
|
-
import org.embulk.spi.TransactionalFileOutput;
|
24
|
-
import org.slf4j.Logger;
|
25
|
-
|
26
|
-
import com.amazonaws.ClientConfiguration;
|
27
|
-
import com.amazonaws.auth.
|
28
|
-
import com.amazonaws.auth.
|
29
|
-
import com.amazonaws.
|
30
|
-
import com.amazonaws.services.s3.
|
31
|
-
import com.
|
32
|
-
|
33
|
-
public class S3FileOutputPlugin implements FileOutputPlugin {
|
34
|
-
public interface PluginTask extends Task {
|
35
|
-
@Config("path_prefix")
|
36
|
-
public String getPathPrefix();
|
37
|
-
|
38
|
-
@Config("file_ext")
|
39
|
-
public String getFileNameExtension();
|
40
|
-
|
41
|
-
@Config("sequence_format")
|
42
|
-
@ConfigDefault("\".%03d.%02d\"")
|
43
|
-
public String getSequenceFormat();
|
44
|
-
|
45
|
-
@Config("bucket")
|
46
|
-
public String getBucket();
|
47
|
-
|
48
|
-
@Config("endpoint")
|
49
|
-
public String getEndpoint();
|
50
|
-
|
51
|
-
@Config("access_key_id")
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
private final
|
67
|
-
|
68
|
-
private final String
|
69
|
-
private final String
|
70
|
-
private final String
|
71
|
-
|
72
|
-
private
|
73
|
-
|
74
|
-
private
|
75
|
-
private
|
76
|
-
private
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
this.
|
109
|
-
this.
|
110
|
-
this.
|
111
|
-
this.
|
112
|
-
this.
|
113
|
-
this.
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
tempFilePath
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
current
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.io.OutputStream;
|
5
|
+
import java.nio.file.Files;
|
6
|
+
import java.nio.file.Path;
|
7
|
+
import java.util.IllegalFormatException;
|
8
|
+
import java.util.List;
|
9
|
+
import java.util.Locale;
|
10
|
+
|
11
|
+
import org.embulk.config.CommitReport;
|
12
|
+
import org.embulk.config.Config;
|
13
|
+
import org.embulk.config.ConfigDefault;
|
14
|
+
import org.embulk.config.ConfigDiff;
|
15
|
+
import org.embulk.config.ConfigException;
|
16
|
+
import org.embulk.config.ConfigSource;
|
17
|
+
import org.embulk.config.Task;
|
18
|
+
import org.embulk.config.TaskSource;
|
19
|
+
import org.embulk.spi.Buffer;
|
20
|
+
import org.embulk.spi.Exec;
|
21
|
+
import org.embulk.spi.FileOutput;
|
22
|
+
import org.embulk.spi.FileOutputPlugin;
|
23
|
+
import org.embulk.spi.TransactionalFileOutput;
|
24
|
+
import org.slf4j.Logger;
|
25
|
+
|
26
|
+
import com.amazonaws.ClientConfiguration;
|
27
|
+
import com.amazonaws.auth.BasicAWSCredentials;
|
28
|
+
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
|
29
|
+
import com.amazonaws.services.s3.AmazonS3Client;
|
30
|
+
import com.amazonaws.services.s3.model.PutObjectRequest;
|
31
|
+
import com.google.common.base.Optional;
|
32
|
+
|
33
|
+
public class S3FileOutputPlugin implements FileOutputPlugin {
|
34
|
+
public interface PluginTask extends Task {
|
35
|
+
@Config("path_prefix")
|
36
|
+
public String getPathPrefix();
|
37
|
+
|
38
|
+
@Config("file_ext")
|
39
|
+
public String getFileNameExtension();
|
40
|
+
|
41
|
+
@Config("sequence_format")
|
42
|
+
@ConfigDefault("\".%03d.%02d\"")
|
43
|
+
public String getSequenceFormat();
|
44
|
+
|
45
|
+
@Config("bucket")
|
46
|
+
public String getBucket();
|
47
|
+
|
48
|
+
@Config("endpoint")
|
49
|
+
public String getEndpoint();
|
50
|
+
|
51
|
+
@Config("access_key_id")
|
52
|
+
@ConfigDefault("null")
|
53
|
+
public Optional<String> getAccessKeyId();
|
54
|
+
|
55
|
+
@Config("secret_access_key")
|
56
|
+
@ConfigDefault("null")
|
57
|
+
public Optional<String> getSecretAccessKey();
|
58
|
+
|
59
|
+
@Config("tmp_path_prefix")
|
60
|
+
@ConfigDefault("\"embulk-output-s3-\"")
|
61
|
+
public String getTempPathPrefix();
|
62
|
+
}
|
63
|
+
|
64
|
+
public static class S3FileOutput implements FileOutput,
|
65
|
+
TransactionalFileOutput {
|
66
|
+
private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
|
67
|
+
|
68
|
+
private final String bucket;
|
69
|
+
private final String pathPrefix;
|
70
|
+
private final String sequenceFormat;
|
71
|
+
private final String fileNameExtension;
|
72
|
+
private final String tempPathPrefix;
|
73
|
+
|
74
|
+
private int taskIndex;
|
75
|
+
private int fileIndex;
|
76
|
+
private AmazonS3Client client;
|
77
|
+
private OutputStream current;
|
78
|
+
private Path tempFilePath;
|
79
|
+
|
80
|
+
private static AmazonS3Client newS3Client(PluginTask task) {
|
81
|
+
AmazonS3Client client = null;
|
82
|
+
try {
|
83
|
+
if (task.getAccessKeyId().isPresent()) {
|
84
|
+
BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
|
85
|
+
task.getAccessKeyId().get(), task.getSecretAccessKey().get());
|
86
|
+
|
87
|
+
ClientConfiguration config = new ClientConfiguration();
|
88
|
+
// TODO: Support more configurations.
|
89
|
+
|
90
|
+
client = new AmazonS3Client(basicAWSCredentials, config);
|
91
|
+
} else {
|
92
|
+
if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
|
93
|
+
client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
|
94
|
+
} else { // IAM ROLE
|
95
|
+
client = new AmazonS3Client();
|
96
|
+
}
|
97
|
+
}
|
98
|
+
client.setEndpoint(task.getEndpoint());
|
99
|
+
client.isRequesterPaysEnabled(task.getBucket()); // check s3 access.
|
100
|
+
} catch (Exception e) {
|
101
|
+
throw new RuntimeException("can't call S3 API. Please check your access_key_id / secret_access_key or s3_region configuration.", e);
|
102
|
+
}
|
103
|
+
|
104
|
+
return client;
|
105
|
+
}
|
106
|
+
|
107
|
+
public S3FileOutput(PluginTask task, int taskIndex) {
|
108
|
+
this.taskIndex = taskIndex;
|
109
|
+
this.client = newS3Client(task);
|
110
|
+
this.bucket = task.getBucket();
|
111
|
+
this.pathPrefix = task.getPathPrefix();
|
112
|
+
this.sequenceFormat = task.getSequenceFormat();
|
113
|
+
this.fileNameExtension = task.getFileNameExtension();
|
114
|
+
this.tempPathPrefix = task.getTempPathPrefix();
|
115
|
+
}
|
116
|
+
|
117
|
+
private static Path newTempFile(String prefix) throws IOException {
|
118
|
+
return Files.createTempFile(prefix, null);
|
119
|
+
}
|
120
|
+
|
121
|
+
private void deleteTempFile() {
|
122
|
+
if (tempFilePath == null) {
|
123
|
+
return;
|
124
|
+
}
|
125
|
+
|
126
|
+
try {
|
127
|
+
Files.delete(tempFilePath);
|
128
|
+
tempFilePath = null;
|
129
|
+
} catch (IOException e) {
|
130
|
+
throw new RuntimeException(e);
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
private String buildCurrentKey() {
|
135
|
+
String sequence = String.format(sequenceFormat, taskIndex,
|
136
|
+
fileIndex);
|
137
|
+
return pathPrefix + sequence + fileNameExtension;
|
138
|
+
}
|
139
|
+
|
140
|
+
private void putFile(Path from, String key) {
|
141
|
+
PutObjectRequest request = new PutObjectRequest(bucket, key,
|
142
|
+
from.toFile());
|
143
|
+
client.putObject(request);
|
144
|
+
}
|
145
|
+
|
146
|
+
private void closeCurrent() {
|
147
|
+
if (current == null) {
|
148
|
+
return;
|
149
|
+
}
|
150
|
+
|
151
|
+
try {
|
152
|
+
putFile(tempFilePath, buildCurrentKey());
|
153
|
+
fileIndex++;
|
154
|
+
} finally {
|
155
|
+
try {
|
156
|
+
current.close();
|
157
|
+
current = null;
|
158
|
+
} catch (IOException e) {
|
159
|
+
throw new RuntimeException(e);
|
160
|
+
} finally {
|
161
|
+
deleteTempFile();
|
162
|
+
}
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
166
|
+
@Override
|
167
|
+
public void nextFile() {
|
168
|
+
closeCurrent();
|
169
|
+
|
170
|
+
try {
|
171
|
+
tempFilePath = newTempFile(tempPathPrefix);
|
172
|
+
|
173
|
+
log.info("Writing S3 file '{}'", buildCurrentKey());
|
174
|
+
|
175
|
+
current = Files.newOutputStream(tempFilePath);
|
176
|
+
} catch (IOException e) {
|
177
|
+
throw new RuntimeException(e);
|
178
|
+
}
|
179
|
+
}
|
180
|
+
|
181
|
+
@Override
|
182
|
+
public void add(Buffer buffer) {
|
183
|
+
if (current == null) {
|
184
|
+
throw new IllegalStateException(
|
185
|
+
"nextFile() must be called before poll()");
|
186
|
+
}
|
187
|
+
|
188
|
+
try {
|
189
|
+
current.write(buffer.array(), buffer.offset(), buffer.limit());
|
190
|
+
} catch (IOException ex) {
|
191
|
+
throw new RuntimeException(ex);
|
192
|
+
} finally {
|
193
|
+
buffer.release();
|
194
|
+
}
|
195
|
+
}
|
196
|
+
|
197
|
+
@Override
|
198
|
+
public void finish() {
|
199
|
+
closeCurrent();
|
200
|
+
}
|
201
|
+
|
202
|
+
@Override
|
203
|
+
public void close() {
|
204
|
+
closeCurrent();
|
205
|
+
}
|
206
|
+
|
207
|
+
@Override
|
208
|
+
public void abort() {
|
209
|
+
deleteTempFile();
|
210
|
+
}
|
211
|
+
|
212
|
+
@Override
|
213
|
+
public CommitReport commit() {
|
214
|
+
CommitReport report = Exec.newCommitReport();
|
215
|
+
return report;
|
216
|
+
}
|
217
|
+
}
|
218
|
+
|
219
|
+
private void validateSequenceFormat(PluginTask task) {
|
220
|
+
try {
|
221
|
+
@SuppressWarnings("unused")
|
222
|
+
String dontCare = String.format(Locale.ENGLISH,
|
223
|
+
task.getSequenceFormat(), 0, 0);
|
224
|
+
} catch (IllegalFormatException ex) {
|
225
|
+
throw new ConfigException(
|
226
|
+
"Invalid sequence_format: parameter for file output plugin",
|
227
|
+
ex);
|
228
|
+
}
|
229
|
+
}
|
230
|
+
|
231
|
+
@Override
|
232
|
+
public ConfigDiff transaction(ConfigSource config, int taskCount,
|
233
|
+
Control control) {
|
234
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
235
|
+
|
236
|
+
validateSequenceFormat(task);
|
237
|
+
|
238
|
+
return resume(task.dump(), taskCount, control);
|
239
|
+
}
|
240
|
+
|
241
|
+
@Override
|
242
|
+
public ConfigDiff resume(TaskSource taskSource, int taskCount,
|
243
|
+
Control control) {
|
244
|
+
control.run(taskSource);
|
245
|
+
return Exec.newConfigDiff();
|
246
|
+
}
|
247
|
+
|
248
|
+
@Override
|
249
|
+
public void cleanup(TaskSource taskSource, int taskCount,
|
250
|
+
List<CommitReport> successCommitReports) {
|
251
|
+
}
|
252
|
+
|
253
|
+
@Override
|
254
|
+
public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
|
255
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
256
|
+
|
257
|
+
return new S3FileOutput(task, taskIndex);
|
258
|
+
}
|
259
|
+
}
|
@@ -1,5 +1,5 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
public class TestS3FileOutputPlugin
|
4
|
-
{
|
5
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
public class TestS3FileOutputPlugin
|
4
|
+
{
|
5
|
+
}
|
metadata
CHANGED
@@ -1,44 +1,44 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Takayama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
41
|
-
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: Stores files on S3.
|
42
42
|
email:
|
43
43
|
- learn.libra@gmail.com
|
44
44
|
executables: []
|
@@ -61,9 +61,10 @@ files:
|
|
61
61
|
- classpath/aws-java-sdk-s3-1.9.24.jar
|
62
62
|
- classpath/commons-codec-1.6.jar
|
63
63
|
- classpath/commons-logging-1.1.3.jar
|
64
|
-
- classpath/embulk-output-s3-0.
|
64
|
+
- classpath/embulk-output-s3-0.2.0.jar
|
65
65
|
- classpath/httpclient-4.3.4.jar
|
66
66
|
- classpath/httpcore-4.3.2.jar
|
67
|
+
- classpath/joda-time-2.8.2.jar
|
67
68
|
homepage: https://github.com/llibra/embulk-output-s3
|
68
69
|
licenses:
|
69
70
|
- MIT
|
Binary file
|