embulk-output-s3 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +11 -11
- data/README.md +39 -39
- data/build.gradle +56 -56
- data/classpath/embulk-output-s3-0.2.0.jar +0 -0
- data/classpath/joda-time-2.8.2.jar +0 -0
- data/src/main/java/org/embulk/output/S3FileOutputPlugin.java +259 -258
- data/src/test/java/org/embulk/output/TestS3FileOutputPlugin.java +5 -5
- metadata +15 -14
- data/classpath/embulk-output-s3-0.1.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9da3670f4971d655c13d7a81cd908cc510a86b6f
|
4
|
+
data.tar.gz: 7e0bb6615000fd7a091057ccb33b385d3773e729
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7cc3cf1287546ba8a40d69114146146ae1031ee5786789f072a3e80b6e1a682d0eabfe6504ac2abde1ac59deef0b557ccafe74899db8efb4dad6ccb1a1522028
|
7
|
+
data.tar.gz: 46e4f2a4a1a5b958f09e93e9ffc7299bfa6b0afbe6c5d7293b56fcb6083fb4b2a86019dcd15c29a27db3fbce9a9b105051f1f4ed2fb8a79702f2302672e5143b
|
data/.gitignore
CHANGED
@@ -1,11 +1,11 @@
|
|
1
|
-
*~
|
2
|
-
/pkg/
|
3
|
-
/tmp/
|
4
|
-
.gradle/
|
5
|
-
/classpath/
|
6
|
-
build/
|
7
|
-
.idea
|
8
|
-
.settings/
|
9
|
-
bin/
|
10
|
-
.classpath
|
11
|
-
.project
|
1
|
+
*~
|
2
|
+
/pkg/
|
3
|
+
/tmp/
|
4
|
+
.gradle/
|
5
|
+
/classpath/
|
6
|
+
build/
|
7
|
+
.idea
|
8
|
+
.settings/
|
9
|
+
bin/
|
10
|
+
.classpath
|
11
|
+
.project
|
data/README.md
CHANGED
@@ -1,39 +1,39 @@
|
|
1
|
-
# S3 file output plugin for Embulk
|
2
|
-
|
3
|
-
## Overview
|
4
|
-
|
5
|
-
* **Plugin type**: file output
|
6
|
-
* **Load all or nothing**: no
|
7
|
-
* **Resume supported**: yes
|
8
|
-
* **Cleanup supported**: yes
|
9
|
-
|
10
|
-
## Configuration
|
11
|
-
|
12
|
-
- **path_prefix**: prefix of target keys (string, required)
|
13
|
-
- **file_ext**: suffix of target keys (string, required)
|
14
|
-
- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
|
15
|
-
- **bucket**: S3 bucket name (string, required)
|
16
|
-
- **endpoint**: S3 endpoint login user name (string, optional)
|
17
|
-
- **access_key_id**: AWS access key id (string,
|
18
|
-
- **secret_access_key**: AWS secret key (string,
|
19
|
-
- **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
|
20
|
-
|
21
|
-
## Example
|
22
|
-
|
23
|
-
```yaml
|
24
|
-
path_prefix: logs/out
|
25
|
-
file_ext: .csv
|
26
|
-
bucket: my-s3-bucket
|
27
|
-
endpoint: s3-us-west-1.amazonaws.com
|
28
|
-
access_key_id: ABCXYZ123ABCXYZ123
|
29
|
-
secret_access_key: AbCxYz123aBcXyZ123
|
30
|
-
formatter:
|
31
|
-
type: csv
|
32
|
-
```
|
33
|
-
|
34
|
-
|
35
|
-
## Build
|
36
|
-
|
37
|
-
```
|
38
|
-
$ ./gradlew gem
|
39
|
-
```
|
1
|
+
# S3 file output plugin for Embulk
|
2
|
+
|
3
|
+
## Overview
|
4
|
+
|
5
|
+
* **Plugin type**: file output
|
6
|
+
* **Load all or nothing**: no
|
7
|
+
* **Resume supported**: yes
|
8
|
+
* **Cleanup supported**: yes
|
9
|
+
|
10
|
+
## Configuration
|
11
|
+
|
12
|
+
- **path_prefix**: prefix of target keys (string, required)
|
13
|
+
- **file_ext**: suffix of target keys (string, required)
|
14
|
+
- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
|
15
|
+
- **bucket**: S3 bucket name (string, required)
|
16
|
+
- **endpoint**: S3 endpoint login user name (string, optional)
|
17
|
+
- **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
18
|
+
- **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
19
|
+
- **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
|
20
|
+
|
21
|
+
## Example
|
22
|
+
|
23
|
+
```yaml
|
24
|
+
path_prefix: logs/out
|
25
|
+
file_ext: .csv
|
26
|
+
bucket: my-s3-bucket
|
27
|
+
endpoint: s3-us-west-1.amazonaws.com
|
28
|
+
access_key_id: ABCXYZ123ABCXYZ123
|
29
|
+
secret_access_key: AbCxYz123aBcXyZ123
|
30
|
+
formatter:
|
31
|
+
type: csv
|
32
|
+
```
|
33
|
+
|
34
|
+
|
35
|
+
## Build
|
36
|
+
|
37
|
+
```
|
38
|
+
$ ./gradlew gem
|
39
|
+
```
|
data/build.gradle
CHANGED
@@ -1,56 +1,56 @@
|
|
1
|
-
plugins {
|
2
|
-
id "com.jfrog.bintray" version "1.1"
|
3
|
-
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
-
id "java"
|
5
|
-
}
|
6
|
-
import com.github.jrubygradle.JRubyExec
|
7
|
-
repositories {
|
8
|
-
mavenCentral()
|
9
|
-
jcenter()
|
10
|
-
}
|
11
|
-
configurations {
|
12
|
-
provided
|
13
|
-
}
|
14
|
-
|
15
|
-
version = "0.
|
16
|
-
|
17
|
-
dependencies {
|
18
|
-
compile "org.embulk:embulk-core:0.5.2"
|
19
|
-
provided "org.embulk:embulk-core:0.5.2"
|
20
|
-
compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
|
21
|
-
testCompile "junit:junit:4.+"
|
22
|
-
}
|
23
|
-
|
24
|
-
task classpath(type: Copy, dependsOn: ["jar"]) {
|
25
|
-
doFirst { file("classpath").deleteDir() }
|
26
|
-
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
27
|
-
into "classpath"
|
28
|
-
}
|
29
|
-
clean { delete 'classpath' }
|
30
|
-
|
31
|
-
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
32
|
-
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
33
|
-
script "build/gemspec"
|
34
|
-
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
35
|
-
}
|
36
|
-
|
37
|
-
task gemspec << { file("build/gemspec").write($/
|
38
|
-
Gem::Specification.new do |spec|
|
39
|
-
spec.name = "${project.name}"
|
40
|
-
spec.version = "${project.version}"
|
41
|
-
spec.authors = ["Manabu Takayama"]
|
42
|
-
spec.summary = %[S3 file output plugin for Embulk]
|
43
|
-
spec.description = %[
|
44
|
-
spec.email = ["learn.libra@gmail.com"]
|
45
|
-
spec.licenses = ["MIT"]
|
46
|
-
spec.homepage = "https://github.com/llibra/embulk-output-s3"
|
47
|
-
|
48
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
49
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
50
|
-
spec.require_paths = ["lib"]
|
51
|
-
|
52
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
53
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
54
|
-
end
|
55
|
-
/$)
|
56
|
-
}
|
1
|
+
plugins {
|
2
|
+
id "com.jfrog.bintray" version "1.1"
|
3
|
+
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "java"
|
5
|
+
}
|
6
|
+
import com.github.jrubygradle.JRubyExec
|
7
|
+
repositories {
|
8
|
+
mavenCentral()
|
9
|
+
jcenter()
|
10
|
+
}
|
11
|
+
configurations {
|
12
|
+
provided
|
13
|
+
}
|
14
|
+
|
15
|
+
version = "0.2.0"
|
16
|
+
|
17
|
+
dependencies {
|
18
|
+
compile "org.embulk:embulk-core:0.5.2"
|
19
|
+
provided "org.embulk:embulk-core:0.5.2"
|
20
|
+
compile "com.amazonaws:aws-java-sdk-s3:1.9.24"
|
21
|
+
testCompile "junit:junit:4.+"
|
22
|
+
}
|
23
|
+
|
24
|
+
task classpath(type: Copy, dependsOn: ["jar"]) {
|
25
|
+
doFirst { file("classpath").deleteDir() }
|
26
|
+
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
27
|
+
into "classpath"
|
28
|
+
}
|
29
|
+
clean { delete 'classpath' }
|
30
|
+
|
31
|
+
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
32
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
33
|
+
script "build/gemspec"
|
34
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
35
|
+
}
|
36
|
+
|
37
|
+
task gemspec << { file("build/gemspec").write($/
|
38
|
+
Gem::Specification.new do |spec|
|
39
|
+
spec.name = "${project.name}"
|
40
|
+
spec.version = "${project.version}"
|
41
|
+
spec.authors = ["Manabu Takayama"]
|
42
|
+
spec.summary = %[S3 file output plugin for Embulk]
|
43
|
+
spec.description = %[Stores files on S3.]
|
44
|
+
spec.email = ["learn.libra@gmail.com"]
|
45
|
+
spec.licenses = ["MIT"]
|
46
|
+
spec.homepage = "https://github.com/llibra/embulk-output-s3"
|
47
|
+
|
48
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
49
|
+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
50
|
+
spec.require_paths = ["lib"]
|
51
|
+
|
52
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
53
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
54
|
+
end
|
55
|
+
/$)
|
56
|
+
}
|
Binary file
|
Binary file
|
@@ -1,258 +1,259 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import java.io.IOException;
|
4
|
-
import java.io.OutputStream;
|
5
|
-
import java.nio.file.Files;
|
6
|
-
import java.nio.file.Path;
|
7
|
-
import java.util.IllegalFormatException;
|
8
|
-
import java.util.List;
|
9
|
-
import java.util.Locale;
|
10
|
-
|
11
|
-
import org.embulk.config.CommitReport;
|
12
|
-
import org.embulk.config.Config;
|
13
|
-
import org.embulk.config.ConfigDefault;
|
14
|
-
import org.embulk.config.ConfigDiff;
|
15
|
-
import org.embulk.config.ConfigException;
|
16
|
-
import org.embulk.config.ConfigSource;
|
17
|
-
import org.embulk.config.Task;
|
18
|
-
import org.embulk.config.TaskSource;
|
19
|
-
import org.embulk.spi.Buffer;
|
20
|
-
import org.embulk.spi.Exec;
|
21
|
-
import org.embulk.spi.FileOutput;
|
22
|
-
import org.embulk.spi.FileOutputPlugin;
|
23
|
-
import org.embulk.spi.TransactionalFileOutput;
|
24
|
-
import org.slf4j.Logger;
|
25
|
-
|
26
|
-
import com.amazonaws.ClientConfiguration;
|
27
|
-
import com.amazonaws.auth.
|
28
|
-
import com.amazonaws.auth.
|
29
|
-
import com.amazonaws.
|
30
|
-
import com.amazonaws.services.s3.
|
31
|
-
import com.
|
32
|
-
|
33
|
-
public class S3FileOutputPlugin implements FileOutputPlugin {
|
34
|
-
public interface PluginTask extends Task {
|
35
|
-
@Config("path_prefix")
|
36
|
-
public String getPathPrefix();
|
37
|
-
|
38
|
-
@Config("file_ext")
|
39
|
-
public String getFileNameExtension();
|
40
|
-
|
41
|
-
@Config("sequence_format")
|
42
|
-
@ConfigDefault("\".%03d.%02d\"")
|
43
|
-
public String getSequenceFormat();
|
44
|
-
|
45
|
-
@Config("bucket")
|
46
|
-
public String getBucket();
|
47
|
-
|
48
|
-
@Config("endpoint")
|
49
|
-
public String getEndpoint();
|
50
|
-
|
51
|
-
@Config("access_key_id")
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
private final
|
67
|
-
|
68
|
-
private final String
|
69
|
-
private final String
|
70
|
-
private final String
|
71
|
-
|
72
|
-
private
|
73
|
-
|
74
|
-
private
|
75
|
-
private
|
76
|
-
private
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
this.
|
109
|
-
this.
|
110
|
-
this.
|
111
|
-
this.
|
112
|
-
this.
|
113
|
-
this.
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
tempFilePath
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
current
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import java.io.IOException;
|
4
|
+
import java.io.OutputStream;
|
5
|
+
import java.nio.file.Files;
|
6
|
+
import java.nio.file.Path;
|
7
|
+
import java.util.IllegalFormatException;
|
8
|
+
import java.util.List;
|
9
|
+
import java.util.Locale;
|
10
|
+
|
11
|
+
import org.embulk.config.CommitReport;
|
12
|
+
import org.embulk.config.Config;
|
13
|
+
import org.embulk.config.ConfigDefault;
|
14
|
+
import org.embulk.config.ConfigDiff;
|
15
|
+
import org.embulk.config.ConfigException;
|
16
|
+
import org.embulk.config.ConfigSource;
|
17
|
+
import org.embulk.config.Task;
|
18
|
+
import org.embulk.config.TaskSource;
|
19
|
+
import org.embulk.spi.Buffer;
|
20
|
+
import org.embulk.spi.Exec;
|
21
|
+
import org.embulk.spi.FileOutput;
|
22
|
+
import org.embulk.spi.FileOutputPlugin;
|
23
|
+
import org.embulk.spi.TransactionalFileOutput;
|
24
|
+
import org.slf4j.Logger;
|
25
|
+
|
26
|
+
import com.amazonaws.ClientConfiguration;
|
27
|
+
import com.amazonaws.auth.BasicAWSCredentials;
|
28
|
+
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
|
29
|
+
import com.amazonaws.services.s3.AmazonS3Client;
|
30
|
+
import com.amazonaws.services.s3.model.PutObjectRequest;
|
31
|
+
import com.google.common.base.Optional;
|
32
|
+
|
33
|
+
public class S3FileOutputPlugin implements FileOutputPlugin {
|
34
|
+
public interface PluginTask extends Task {
|
35
|
+
@Config("path_prefix")
|
36
|
+
public String getPathPrefix();
|
37
|
+
|
38
|
+
@Config("file_ext")
|
39
|
+
public String getFileNameExtension();
|
40
|
+
|
41
|
+
@Config("sequence_format")
|
42
|
+
@ConfigDefault("\".%03d.%02d\"")
|
43
|
+
public String getSequenceFormat();
|
44
|
+
|
45
|
+
@Config("bucket")
|
46
|
+
public String getBucket();
|
47
|
+
|
48
|
+
@Config("endpoint")
|
49
|
+
public String getEndpoint();
|
50
|
+
|
51
|
+
@Config("access_key_id")
|
52
|
+
@ConfigDefault("null")
|
53
|
+
public Optional<String> getAccessKeyId();
|
54
|
+
|
55
|
+
@Config("secret_access_key")
|
56
|
+
@ConfigDefault("null")
|
57
|
+
public Optional<String> getSecretAccessKey();
|
58
|
+
|
59
|
+
@Config("tmp_path_prefix")
|
60
|
+
@ConfigDefault("\"embulk-output-s3-\"")
|
61
|
+
public String getTempPathPrefix();
|
62
|
+
}
|
63
|
+
|
64
|
+
public static class S3FileOutput implements FileOutput,
|
65
|
+
TransactionalFileOutput {
|
66
|
+
private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
|
67
|
+
|
68
|
+
private final String bucket;
|
69
|
+
private final String pathPrefix;
|
70
|
+
private final String sequenceFormat;
|
71
|
+
private final String fileNameExtension;
|
72
|
+
private final String tempPathPrefix;
|
73
|
+
|
74
|
+
private int taskIndex;
|
75
|
+
private int fileIndex;
|
76
|
+
private AmazonS3Client client;
|
77
|
+
private OutputStream current;
|
78
|
+
private Path tempFilePath;
|
79
|
+
|
80
|
+
private static AmazonS3Client newS3Client(PluginTask task) {
|
81
|
+
AmazonS3Client client = null;
|
82
|
+
try {
|
83
|
+
if (task.getAccessKeyId().isPresent()) {
|
84
|
+
BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
|
85
|
+
task.getAccessKeyId().get(), task.getSecretAccessKey().get());
|
86
|
+
|
87
|
+
ClientConfiguration config = new ClientConfiguration();
|
88
|
+
// TODO: Support more configurations.
|
89
|
+
|
90
|
+
client = new AmazonS3Client(basicAWSCredentials, config);
|
91
|
+
} else {
|
92
|
+
if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
|
93
|
+
client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
|
94
|
+
} else { // IAM ROLE
|
95
|
+
client = new AmazonS3Client();
|
96
|
+
}
|
97
|
+
}
|
98
|
+
client.setEndpoint(task.getEndpoint());
|
99
|
+
client.isRequesterPaysEnabled(task.getBucket()); // check s3 access.
|
100
|
+
} catch (Exception e) {
|
101
|
+
throw new RuntimeException("can't call S3 API. Please check your access_key_id / secret_access_key or s3_region configuration.", e);
|
102
|
+
}
|
103
|
+
|
104
|
+
return client;
|
105
|
+
}
|
106
|
+
|
107
|
+
public S3FileOutput(PluginTask task, int taskIndex) {
|
108
|
+
this.taskIndex = taskIndex;
|
109
|
+
this.client = newS3Client(task);
|
110
|
+
this.bucket = task.getBucket();
|
111
|
+
this.pathPrefix = task.getPathPrefix();
|
112
|
+
this.sequenceFormat = task.getSequenceFormat();
|
113
|
+
this.fileNameExtension = task.getFileNameExtension();
|
114
|
+
this.tempPathPrefix = task.getTempPathPrefix();
|
115
|
+
}
|
116
|
+
|
117
|
+
private static Path newTempFile(String prefix) throws IOException {
|
118
|
+
return Files.createTempFile(prefix, null);
|
119
|
+
}
|
120
|
+
|
121
|
+
private void deleteTempFile() {
|
122
|
+
if (tempFilePath == null) {
|
123
|
+
return;
|
124
|
+
}
|
125
|
+
|
126
|
+
try {
|
127
|
+
Files.delete(tempFilePath);
|
128
|
+
tempFilePath = null;
|
129
|
+
} catch (IOException e) {
|
130
|
+
throw new RuntimeException(e);
|
131
|
+
}
|
132
|
+
}
|
133
|
+
|
134
|
+
private String buildCurrentKey() {
|
135
|
+
String sequence = String.format(sequenceFormat, taskIndex,
|
136
|
+
fileIndex);
|
137
|
+
return pathPrefix + sequence + fileNameExtension;
|
138
|
+
}
|
139
|
+
|
140
|
+
private void putFile(Path from, String key) {
|
141
|
+
PutObjectRequest request = new PutObjectRequest(bucket, key,
|
142
|
+
from.toFile());
|
143
|
+
client.putObject(request);
|
144
|
+
}
|
145
|
+
|
146
|
+
private void closeCurrent() {
|
147
|
+
if (current == null) {
|
148
|
+
return;
|
149
|
+
}
|
150
|
+
|
151
|
+
try {
|
152
|
+
putFile(tempFilePath, buildCurrentKey());
|
153
|
+
fileIndex++;
|
154
|
+
} finally {
|
155
|
+
try {
|
156
|
+
current.close();
|
157
|
+
current = null;
|
158
|
+
} catch (IOException e) {
|
159
|
+
throw new RuntimeException(e);
|
160
|
+
} finally {
|
161
|
+
deleteTempFile();
|
162
|
+
}
|
163
|
+
}
|
164
|
+
}
|
165
|
+
|
166
|
+
@Override
|
167
|
+
public void nextFile() {
|
168
|
+
closeCurrent();
|
169
|
+
|
170
|
+
try {
|
171
|
+
tempFilePath = newTempFile(tempPathPrefix);
|
172
|
+
|
173
|
+
log.info("Writing S3 file '{}'", buildCurrentKey());
|
174
|
+
|
175
|
+
current = Files.newOutputStream(tempFilePath);
|
176
|
+
} catch (IOException e) {
|
177
|
+
throw new RuntimeException(e);
|
178
|
+
}
|
179
|
+
}
|
180
|
+
|
181
|
+
@Override
|
182
|
+
public void add(Buffer buffer) {
|
183
|
+
if (current == null) {
|
184
|
+
throw new IllegalStateException(
|
185
|
+
"nextFile() must be called before poll()");
|
186
|
+
}
|
187
|
+
|
188
|
+
try {
|
189
|
+
current.write(buffer.array(), buffer.offset(), buffer.limit());
|
190
|
+
} catch (IOException ex) {
|
191
|
+
throw new RuntimeException(ex);
|
192
|
+
} finally {
|
193
|
+
buffer.release();
|
194
|
+
}
|
195
|
+
}
|
196
|
+
|
197
|
+
@Override
|
198
|
+
public void finish() {
|
199
|
+
closeCurrent();
|
200
|
+
}
|
201
|
+
|
202
|
+
@Override
|
203
|
+
public void close() {
|
204
|
+
closeCurrent();
|
205
|
+
}
|
206
|
+
|
207
|
+
@Override
|
208
|
+
public void abort() {
|
209
|
+
deleteTempFile();
|
210
|
+
}
|
211
|
+
|
212
|
+
@Override
|
213
|
+
public CommitReport commit() {
|
214
|
+
CommitReport report = Exec.newCommitReport();
|
215
|
+
return report;
|
216
|
+
}
|
217
|
+
}
|
218
|
+
|
219
|
+
private void validateSequenceFormat(PluginTask task) {
|
220
|
+
try {
|
221
|
+
@SuppressWarnings("unused")
|
222
|
+
String dontCare = String.format(Locale.ENGLISH,
|
223
|
+
task.getSequenceFormat(), 0, 0);
|
224
|
+
} catch (IllegalFormatException ex) {
|
225
|
+
throw new ConfigException(
|
226
|
+
"Invalid sequence_format: parameter for file output plugin",
|
227
|
+
ex);
|
228
|
+
}
|
229
|
+
}
|
230
|
+
|
231
|
+
@Override
|
232
|
+
public ConfigDiff transaction(ConfigSource config, int taskCount,
|
233
|
+
Control control) {
|
234
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
235
|
+
|
236
|
+
validateSequenceFormat(task);
|
237
|
+
|
238
|
+
return resume(task.dump(), taskCount, control);
|
239
|
+
}
|
240
|
+
|
241
|
+
@Override
|
242
|
+
public ConfigDiff resume(TaskSource taskSource, int taskCount,
|
243
|
+
Control control) {
|
244
|
+
control.run(taskSource);
|
245
|
+
return Exec.newConfigDiff();
|
246
|
+
}
|
247
|
+
|
248
|
+
@Override
|
249
|
+
public void cleanup(TaskSource taskSource, int taskCount,
|
250
|
+
List<CommitReport> successCommitReports) {
|
251
|
+
}
|
252
|
+
|
253
|
+
@Override
|
254
|
+
public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
|
255
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
256
|
+
|
257
|
+
return new S3FileOutput(task, taskIndex);
|
258
|
+
}
|
259
|
+
}
|
@@ -1,5 +1,5 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
public class TestS3FileOutputPlugin
|
4
|
-
{
|
5
|
-
}
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
public class TestS3FileOutputPlugin
|
4
|
+
{
|
5
|
+
}
|
metadata
CHANGED
@@ -1,44 +1,44 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Takayama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-08-29 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
41
|
-
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
description: Stores files on S3.
|
42
42
|
email:
|
43
43
|
- learn.libra@gmail.com
|
44
44
|
executables: []
|
@@ -61,9 +61,10 @@ files:
|
|
61
61
|
- classpath/aws-java-sdk-s3-1.9.24.jar
|
62
62
|
- classpath/commons-codec-1.6.jar
|
63
63
|
- classpath/commons-logging-1.1.3.jar
|
64
|
-
- classpath/embulk-output-s3-0.
|
64
|
+
- classpath/embulk-output-s3-0.2.0.jar
|
65
65
|
- classpath/httpclient-4.3.4.jar
|
66
66
|
- classpath/httpcore-4.3.2.jar
|
67
|
+
- classpath/joda-time-2.8.2.jar
|
67
68
|
homepage: https://github.com/llibra/embulk-output-s3
|
68
69
|
licenses:
|
69
70
|
- MIT
|
Binary file
|