embulk-output-s3 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +13 -11
- data/README.md +60 -39
- data/build.gradle +77 -59
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +6 -6
- data/settings.gradle +1 -0
- data/src/main/java/org/embulk/output/S3FileOutputPlugin.java +301 -258
- metadata +6 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6494f870318029670e7e568288e0acfd63eb0ce0
|
4
|
+
data.tar.gz: fe7baf6a69629a714530a67d2aa2c718e6eb02ae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c0f8a74ea56053b6daa9ffc981a52af413adc47ebd63f56fac609967125927e731e782294d5db93353dd5bf35fbb0b8e640cd76f6c6cc559aa83eec89017290c
|
7
|
+
data.tar.gz: f64f27ad7fd6b21d27843dcb46c486522064915abf42e9058d850248b590520010fcd1d4431f128f3bd30bddab6f44d00ce36b46075c13b533fd122923b58cd8
|
data/.gitignore
CHANGED
@@ -1,11 +1,13 @@
|
|
1
|
-
*~
|
2
|
-
/pkg/
|
3
|
-
/tmp/
|
4
|
-
.gradle/
|
5
|
-
/classpath/
|
6
|
-
build/
|
7
|
-
.idea
|
8
|
-
.settings/
|
9
|
-
bin/
|
10
|
-
.classpath
|
11
|
-
.project
|
1
|
+
*~
|
2
|
+
/pkg/
|
3
|
+
/tmp/
|
4
|
+
.gradle/
|
5
|
+
/classpath/
|
6
|
+
build/
|
7
|
+
.idea
|
8
|
+
.settings/
|
9
|
+
bin/
|
10
|
+
.classpath
|
11
|
+
.project
|
12
|
+
*.gemspec
|
13
|
+
*.iml
|
data/README.md
CHANGED
@@ -1,39 +1,60 @@
|
|
1
|
-
# S3 file output plugin for Embulk
|
2
|
-
|
3
|
-
##
|
4
|
-
|
5
|
-
*
|
6
|
-
*
|
7
|
-
*
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
- **
|
19
|
-
- **
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
1
|
+
# S3 file output plugin for Embulk
|
2
|
+
|
3
|
+
## Developers
|
4
|
+
|
5
|
+
* Manabu Takayama <learn.libra@gmail.com>
|
6
|
+
* toyama hiroshi <toyama0919@gmail.com>
|
7
|
+
* Civitaspo <civitaspo@gmail.com>
|
8
|
+
|
9
|
+
## Overview
|
10
|
+
|
11
|
+
* **Plugin type**: file output
|
12
|
+
* **Load all or nothing**: no
|
13
|
+
* **Resume supported**: yes
|
14
|
+
* **Cleanup supported**: yes
|
15
|
+
|
16
|
+
## Configuration
|
17
|
+
|
18
|
+
- **path_prefix**: prefix of target keys (string, required)
|
19
|
+
- **file_ext**: suffix of target keys (string, required)
|
20
|
+
- **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
|
21
|
+
- **bucket**: S3 bucket name (string, required)
|
22
|
+
- **endpoint**: S3 endpoint login user name (string, optional)
|
23
|
+
- **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
24
|
+
- **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
|
25
|
+
- **tmp_path_prefix**: prefix of temporary files (string, default: 'embulk-output-s3-')
|
26
|
+
- **canned_acl**: canned access control list for created objects ([enum](#cannedaccesscontrollist), default: null)
|
27
|
+
|
28
|
+
### CannedAccessControlList
|
29
|
+
you can choose one of the below list.
|
30
|
+
|
31
|
+
- AuthenticatedRead
|
32
|
+
- AwsExecRead
|
33
|
+
- BucketOwnerFullControl
|
34
|
+
- BucketOwnerRead
|
35
|
+
- LogDeliveryWrite
|
36
|
+
- Private
|
37
|
+
- PublicRead
|
38
|
+
- PublicReadWrite
|
39
|
+
|
40
|
+
cf. http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html
|
41
|
+
|
42
|
+
## Example
|
43
|
+
|
44
|
+
```yaml
|
45
|
+
path_prefix: logs/out
|
46
|
+
file_ext: .csv
|
47
|
+
bucket: my-s3-bucket
|
48
|
+
endpoint: s3-us-west-1.amazonaws.com
|
49
|
+
access_key_id: ABCXYZ123ABCXYZ123
|
50
|
+
secret_access_key: AbCxYz123aBcXyZ123
|
51
|
+
formatter:
|
52
|
+
type: csv
|
53
|
+
```
|
54
|
+
|
55
|
+
|
56
|
+
## Build
|
57
|
+
|
58
|
+
```
|
59
|
+
$ ./gradlew gem
|
60
|
+
```
|
data/build.gradle
CHANGED
@@ -1,59 +1,77 @@
|
|
1
|
-
plugins {
|
2
|
-
id "com.jfrog.bintray" version "1.1"
|
3
|
-
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
-
id "java"
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
}
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
1
|
+
plugins {
|
2
|
+
id "com.jfrog.bintray" version "1.1"
|
3
|
+
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "java"
|
5
|
+
id "checkstyle"
|
6
|
+
}
|
7
|
+
import com.github.jrubygradle.JRubyExec
|
8
|
+
repositories {
|
9
|
+
mavenCentral()
|
10
|
+
jcenter()
|
11
|
+
}
|
12
|
+
configurations {
|
13
|
+
provided
|
14
|
+
}
|
15
|
+
|
16
|
+
version = "1.2.0"
|
17
|
+
|
18
|
+
sourceCompatibility = 1.7
|
19
|
+
targetCompatibility = 1.7
|
20
|
+
|
21
|
+
dependencies {
|
22
|
+
compile "org.embulk:embulk-core:0.8.1"
|
23
|
+
provided "org.embulk:embulk-core:0.8.1"
|
24
|
+
compile "com.amazonaws:aws-java-sdk-s3:1.10.26"
|
25
|
+
testCompile "junit:junit:4.+"
|
26
|
+
}
|
27
|
+
|
28
|
+
task classpath(type: Copy, dependsOn: ["jar"]) {
|
29
|
+
doFirst { file("classpath").deleteDir() }
|
30
|
+
from (configurations.runtime - configurations.provided + files(jar.archivePath))
|
31
|
+
into "classpath"
|
32
|
+
}
|
33
|
+
clean { delete 'classpath' }
|
34
|
+
|
35
|
+
checkstyle {
|
36
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
37
|
+
toolVersion = '6.14.1'
|
38
|
+
}
|
39
|
+
checkstyleMain {
|
40
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
41
|
+
ignoreFailures = true
|
42
|
+
}
|
43
|
+
checkstyleTest {
|
44
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
45
|
+
ignoreFailures = true
|
46
|
+
}
|
47
|
+
task checkstyle(type: Checkstyle) {
|
48
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
49
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
50
|
+
}
|
51
|
+
|
52
|
+
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
53
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
54
|
+
script "build/gemspec"
|
55
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
56
|
+
}
|
57
|
+
|
58
|
+
task gemspec << { file("build/gemspec").write($/
|
59
|
+
Gem::Specification.new do |spec|
|
60
|
+
spec.name = "${project.name}"
|
61
|
+
spec.version = "${project.version}"
|
62
|
+
spec.authors = ["Manabu Takayama"]
|
63
|
+
spec.summary = %[S3 file output plugin for Embulk]
|
64
|
+
spec.description = %[Stores files on S3.]
|
65
|
+
spec.email = ["learn.libra@gmail.com"]
|
66
|
+
spec.licenses = ["MIT"]
|
67
|
+
spec.homepage = "https://github.com/llibra/embulk-output-s3"
|
68
|
+
|
69
|
+
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
70
|
+
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
71
|
+
spec.require_paths = ["lib"]
|
72
|
+
|
73
|
+
spec.add_development_dependency 'bundler', ['~> 1.0']
|
74
|
+
spec.add_development_dependency 'rake', ['>= 10.0']
|
75
|
+
end
|
76
|
+
/$)
|
77
|
+
}
|
@@ -0,0 +1,128 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<!DOCTYPE module PUBLIC
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
5
|
+
<module name="Checker">
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
7
|
+
<module name="FileTabCharacter"/>
|
8
|
+
<module name="NewlineAtEndOfFile">
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
10
|
+
</module>
|
11
|
+
<module name="RegexpMultiline">
|
12
|
+
<property name="format" value="\r"/>
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
14
|
+
</module>
|
15
|
+
<module name="RegexpMultiline">
|
16
|
+
<property name="format" value=" \n"/>
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
18
|
+
</module>
|
19
|
+
<module name="RegexpMultiline">
|
20
|
+
<property name="format" value="\{\n\n"/>
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
22
|
+
</module>
|
23
|
+
<module name="RegexpMultiline">
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
26
|
+
</module>
|
27
|
+
<module name="RegexpMultiline">
|
28
|
+
<property name="format" value="\n\n\n"/>
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
30
|
+
</module>
|
31
|
+
<module name="RegexpMultiline">
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
34
|
+
</module>
|
35
|
+
<module name="RegexpMultiline">
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
38
|
+
</module>
|
39
|
+
|
40
|
+
<module name="TreeWalker">
|
41
|
+
<module name="EmptyBlock">
|
42
|
+
<property name="option" value="text"/>
|
43
|
+
<property name="tokens" value="
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
46
|
+
</module>
|
47
|
+
<module name="EmptyStatement"/>
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
49
|
+
<module name="EmptyForIteratorPad">
|
50
|
+
<property name="option" value="space"/>
|
51
|
+
</module>
|
52
|
+
<module name="MethodParamPad">
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
54
|
+
<property name="option" value="nospace"/>
|
55
|
+
</module>
|
56
|
+
<module name="ParenPad"/>
|
57
|
+
<module name="TypecastParenPad"/>
|
58
|
+
<module name="NeedBraces"/>
|
59
|
+
<module name="LeftCurly">
|
60
|
+
<property name="option" value="nl"/>
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
62
|
+
</module>
|
63
|
+
<module name="LeftCurly">
|
64
|
+
<property name="option" value="eol"/>
|
65
|
+
<property name="tokens" value="
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
68
|
+
</module>
|
69
|
+
<module name="RightCurly">
|
70
|
+
<property name="option" value="alone"/>
|
71
|
+
</module>
|
72
|
+
<module name="GenericWhitespace"/>
|
73
|
+
<module name="WhitespaceAfter"/>
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
75
|
+
|
76
|
+
<module name="UpperEll"/>
|
77
|
+
<module name="DefaultComesLast"/>
|
78
|
+
<module name="ArrayTypeStyle"/>
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
80
|
+
<module name="ModifierOrder"/>
|
81
|
+
<module name="OneStatementPerLine"/>
|
82
|
+
<module name="StringLiteralEquality"/>
|
83
|
+
<module name="MutableException"/>
|
84
|
+
<module name="EqualsHashCode"/>
|
85
|
+
<module name="InnerAssignment"/>
|
86
|
+
<module name="InterfaceIsType"/>
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
88
|
+
|
89
|
+
<module name="MemberName"/>
|
90
|
+
<module name="LocalVariableName"/>
|
91
|
+
<module name="LocalFinalVariableName"/>
|
92
|
+
<module name="TypeName"/>
|
93
|
+
<module name="PackageName"/>
|
94
|
+
<module name="ParameterName"/>
|
95
|
+
<module name="StaticVariableName"/>
|
96
|
+
<module name="ClassTypeParameterName">
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
98
|
+
</module>
|
99
|
+
<module name="MethodTypeParameterName">
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
101
|
+
</module>
|
102
|
+
|
103
|
+
<module name="AvoidStarImport"/>
|
104
|
+
<module name="RedundantImport"/>
|
105
|
+
<module name="UnusedImports"/>
|
106
|
+
<module name="ImportOrder">
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
108
|
+
<property name="separated" value="true"/>
|
109
|
+
<property name="option" value="bottom"/>
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
111
|
+
</module>
|
112
|
+
|
113
|
+
<module name="WhitespaceAround">
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
117
|
+
<property name="tokens" value="
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
126
|
+
</module>
|
127
|
+
</module>
|
128
|
+
</module>
|
@@ -0,0 +1,108 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<!DOCTYPE module PUBLIC
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
5
|
+
<!--
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
7
|
+
-->
|
8
|
+
<module name="Checker">
|
9
|
+
<module name="FileTabCharacter"/>
|
10
|
+
<module name="NewlineAtEndOfFile">
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
12
|
+
</module>
|
13
|
+
<module name="RegexpMultiline">
|
14
|
+
<property name="format" value="\r"/>
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
16
|
+
</module>
|
17
|
+
<module name="RegexpMultiline">
|
18
|
+
<property name="format" value=" \n"/>
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
20
|
+
</module>
|
21
|
+
<module name="RegexpMultiline">
|
22
|
+
<property name="format" value="\n\n\n"/>
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
24
|
+
</module>
|
25
|
+
<module name="RegexpMultiline">
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
28
|
+
</module>
|
29
|
+
|
30
|
+
<module name="TreeWalker">
|
31
|
+
<module name="EmptyBlock">
|
32
|
+
<property name="option" value="text"/>
|
33
|
+
<property name="tokens" value="
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
36
|
+
</module>
|
37
|
+
<module name="EmptyStatement"/>
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
39
|
+
<module name="EmptyForIteratorPad">
|
40
|
+
<property name="option" value="space"/>
|
41
|
+
</module>
|
42
|
+
<module name="MethodParamPad">
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
44
|
+
<property name="option" value="nospace"/>
|
45
|
+
</module>
|
46
|
+
<module name="ParenPad"/>
|
47
|
+
<module name="TypecastParenPad"/>
|
48
|
+
<module name="NeedBraces"/>
|
49
|
+
<module name="LeftCurly">
|
50
|
+
<property name="option" value="nl"/>
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
52
|
+
</module>
|
53
|
+
<module name="LeftCurly">
|
54
|
+
<property name="option" value="eol"/>
|
55
|
+
<property name="tokens" value="
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
58
|
+
</module>
|
59
|
+
<module name="RightCurly">
|
60
|
+
<property name="option" value="alone"/>
|
61
|
+
</module>
|
62
|
+
<module name="GenericWhitespace"/>
|
63
|
+
<module name="WhitespaceAfter"/>
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
65
|
+
|
66
|
+
<module name="UpperEll"/>
|
67
|
+
<module name="DefaultComesLast"/>
|
68
|
+
<module name="ArrayTypeStyle"/>
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
70
|
+
<module name="ModifierOrder"/>
|
71
|
+
<module name="OneStatementPerLine"/>
|
72
|
+
<module name="StringLiteralEquality"/>
|
73
|
+
<module name="MutableException"/>
|
74
|
+
<module name="EqualsHashCode"/>
|
75
|
+
<module name="InnerAssignment"/>
|
76
|
+
<module name="InterfaceIsType"/>
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
78
|
+
|
79
|
+
<module name="MemberName"/>
|
80
|
+
<module name="LocalVariableName"/>
|
81
|
+
<module name="LocalFinalVariableName"/>
|
82
|
+
<module name="TypeName"/>
|
83
|
+
<module name="PackageName"/>
|
84
|
+
<module name="ParameterName"/>
|
85
|
+
<module name="StaticVariableName"/>
|
86
|
+
<module name="ClassTypeParameterName">
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
88
|
+
</module>
|
89
|
+
<module name="MethodTypeParameterName">
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
91
|
+
</module>
|
92
|
+
|
93
|
+
<module name="WhitespaceAround">
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
97
|
+
<property name="tokens" value="
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
106
|
+
</module>
|
107
|
+
</module>
|
108
|
+
</module>
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
2
|
-
distributionBase=GRADLE_USER_HOME
|
3
|
-
distributionPath=wrapper/dists
|
4
|
-
zipStoreBase=GRADLE_USER_HOME
|
5
|
-
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
2
|
+
distributionBase=GRADLE_USER_HOME
|
3
|
+
distributionPath=wrapper/dists
|
4
|
+
zipStoreBase=GRADLE_USER_HOME
|
5
|
+
zipStorePath=wrapper/dists
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
data/settings.gradle
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rootProject.name = 'embulk-output-s3'
|
@@ -1,258 +1,301 @@
|
|
1
|
-
package org.embulk.output;
|
2
|
-
|
3
|
-
import
|
4
|
-
import
|
5
|
-
import
|
6
|
-
import
|
7
|
-
import
|
8
|
-
import
|
9
|
-
import
|
10
|
-
|
11
|
-
import org.embulk.config.
|
12
|
-
import org.embulk.config.
|
13
|
-
import org.embulk.config.
|
14
|
-
import org.embulk.config.
|
15
|
-
import org.embulk.config.
|
16
|
-
import org.embulk.config.
|
17
|
-
import org.embulk.config.
|
18
|
-
import org.embulk.
|
19
|
-
import org.embulk.spi.
|
20
|
-
import org.embulk.spi.
|
21
|
-
import org.embulk.spi.
|
22
|
-
import org.embulk.spi.
|
23
|
-
import org.
|
24
|
-
|
25
|
-
|
26
|
-
import
|
27
|
-
import
|
28
|
-
import
|
29
|
-
import
|
30
|
-
import
|
31
|
-
import
|
32
|
-
|
33
|
-
public class S3FileOutputPlugin
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
@
|
43
|
-
|
44
|
-
|
45
|
-
@Config("
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
@
|
50
|
-
|
51
|
-
|
52
|
-
@Config("
|
53
|
-
@ConfigDefault("null")
|
54
|
-
|
55
|
-
|
56
|
-
@Config("
|
57
|
-
@ConfigDefault("null")
|
58
|
-
|
59
|
-
|
60
|
-
@Config("
|
61
|
-
@ConfigDefault("
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
private
|
78
|
-
|
79
|
-
private
|
80
|
-
|
81
|
-
private
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
return
|
118
|
-
}
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
}
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
1
|
+
package org.embulk.output;
|
2
|
+
|
3
|
+
import com.amazonaws.ClientConfiguration;
|
4
|
+
import com.amazonaws.auth.BasicAWSCredentials;
|
5
|
+
import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
|
6
|
+
import com.amazonaws.services.s3.AmazonS3Client;
|
7
|
+
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
8
|
+
import com.amazonaws.services.s3.model.PutObjectRequest;
|
9
|
+
import com.google.common.base.Optional;
|
10
|
+
import org.embulk.config.Config;
|
11
|
+
import org.embulk.config.ConfigDefault;
|
12
|
+
import org.embulk.config.ConfigDiff;
|
13
|
+
import org.embulk.config.ConfigException;
|
14
|
+
import org.embulk.config.ConfigSource;
|
15
|
+
import org.embulk.config.Task;
|
16
|
+
import org.embulk.config.TaskReport;
|
17
|
+
import org.embulk.config.TaskSource;
|
18
|
+
import org.embulk.spi.Buffer;
|
19
|
+
import org.embulk.spi.Exec;
|
20
|
+
import org.embulk.spi.FileOutput;
|
21
|
+
import org.embulk.spi.FileOutputPlugin;
|
22
|
+
import org.embulk.spi.TransactionalFileOutput;
|
23
|
+
import org.slf4j.Logger;
|
24
|
+
|
25
|
+
import java.io.IOException;
|
26
|
+
import java.io.OutputStream;
|
27
|
+
import java.nio.file.Files;
|
28
|
+
import java.nio.file.Path;
|
29
|
+
import java.util.IllegalFormatException;
|
30
|
+
import java.util.List;
|
31
|
+
import java.util.Locale;
|
32
|
+
|
33
|
+
public class S3FileOutputPlugin
|
34
|
+
implements FileOutputPlugin
|
35
|
+
{
|
36
|
+
public interface PluginTask
|
37
|
+
extends Task
|
38
|
+
{
|
39
|
+
@Config("path_prefix")
|
40
|
+
String getPathPrefix();
|
41
|
+
|
42
|
+
@Config("file_ext")
|
43
|
+
String getFileNameExtension();
|
44
|
+
|
45
|
+
@Config("sequence_format")
|
46
|
+
@ConfigDefault("\".%03d.%02d\"")
|
47
|
+
String getSequenceFormat();
|
48
|
+
|
49
|
+
@Config("bucket")
|
50
|
+
String getBucket();
|
51
|
+
|
52
|
+
@Config("endpoint")
|
53
|
+
@ConfigDefault("null")
|
54
|
+
Optional<String> getEndpoint();
|
55
|
+
|
56
|
+
@Config("access_key_id")
|
57
|
+
@ConfigDefault("null")
|
58
|
+
Optional<String> getAccessKeyId();
|
59
|
+
|
60
|
+
@Config("secret_access_key")
|
61
|
+
@ConfigDefault("null")
|
62
|
+
Optional<String> getSecretAccessKey();
|
63
|
+
|
64
|
+
@Config("tmp_path_prefix")
|
65
|
+
@ConfigDefault("\"embulk-output-s3-\"")
|
66
|
+
String getTempPathPrefix();
|
67
|
+
|
68
|
+
@Config("canned_acl")
|
69
|
+
@ConfigDefault("null")
|
70
|
+
Optional<CannedAccessControlList> getCannedAccessControlList();
|
71
|
+
}
|
72
|
+
|
73
|
+
public static class S3FileOutput
|
74
|
+
implements FileOutput,
|
75
|
+
TransactionalFileOutput
|
76
|
+
{
|
77
|
+
private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
|
78
|
+
|
79
|
+
private final String bucket;
|
80
|
+
private final String pathPrefix;
|
81
|
+
private final String sequenceFormat;
|
82
|
+
private final String fileNameExtension;
|
83
|
+
private final String tempPathPrefix;
|
84
|
+
private final Optional<CannedAccessControlList> cannedAccessControlListOptional;
|
85
|
+
|
86
|
+
private int taskIndex;
|
87
|
+
private int fileIndex;
|
88
|
+
private AmazonS3Client client;
|
89
|
+
private OutputStream current;
|
90
|
+
private Path tempFilePath;
|
91
|
+
|
92
|
+
private static AmazonS3Client newS3Client(PluginTask task)
|
93
|
+
{
|
94
|
+
AmazonS3Client client;
|
95
|
+
|
96
|
+
if (task.getAccessKeyId().isPresent()) {
|
97
|
+
BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
|
98
|
+
task.getAccessKeyId().get(), task.getSecretAccessKey().get());
|
99
|
+
|
100
|
+
ClientConfiguration config = new ClientConfiguration();
|
101
|
+
// TODO: Support more configurations.
|
102
|
+
|
103
|
+
client = new AmazonS3Client(basicAWSCredentials, config);
|
104
|
+
}
|
105
|
+
else {
|
106
|
+
if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
|
107
|
+
client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
|
108
|
+
}
|
109
|
+
else { // IAM ROLE
|
110
|
+
client = new AmazonS3Client();
|
111
|
+
}
|
112
|
+
}
|
113
|
+
if (task.getEndpoint().isPresent()) {
|
114
|
+
client.setEndpoint(task.getEndpoint().get());
|
115
|
+
}
|
116
|
+
|
117
|
+
return client;
|
118
|
+
}
|
119
|
+
|
120
|
+
public S3FileOutput(PluginTask task, int taskIndex)
|
121
|
+
{
|
122
|
+
this.taskIndex = taskIndex;
|
123
|
+
this.client = newS3Client(task);
|
124
|
+
this.bucket = task.getBucket();
|
125
|
+
this.pathPrefix = task.getPathPrefix();
|
126
|
+
this.sequenceFormat = task.getSequenceFormat();
|
127
|
+
this.fileNameExtension = task.getFileNameExtension();
|
128
|
+
this.tempPathPrefix = task.getTempPathPrefix();
|
129
|
+
this.cannedAccessControlListOptional = task.getCannedAccessControlList();
|
130
|
+
}
|
131
|
+
|
132
|
+
private static Path newTempFile(String prefix)
|
133
|
+
throws IOException
|
134
|
+
{
|
135
|
+
return Files.createTempFile(prefix, null);
|
136
|
+
}
|
137
|
+
|
138
|
+
private void deleteTempFile()
|
139
|
+
{
|
140
|
+
if (tempFilePath == null) {
|
141
|
+
return;
|
142
|
+
}
|
143
|
+
|
144
|
+
try {
|
145
|
+
Files.delete(tempFilePath);
|
146
|
+
tempFilePath = null;
|
147
|
+
}
|
148
|
+
catch (IOException e) {
|
149
|
+
throw new RuntimeException(e);
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
private String buildCurrentKey()
|
154
|
+
{
|
155
|
+
String sequence = String.format(sequenceFormat, taskIndex,
|
156
|
+
fileIndex);
|
157
|
+
return pathPrefix + sequence + fileNameExtension;
|
158
|
+
}
|
159
|
+
|
160
|
+
private void putFile(Path from, String key)
|
161
|
+
{
|
162
|
+
PutObjectRequest request = new PutObjectRequest(bucket, key, from.toFile());
|
163
|
+
if (cannedAccessControlListOptional.isPresent()) {
|
164
|
+
request.withCannedAcl(cannedAccessControlListOptional.get());
|
165
|
+
}
|
166
|
+
client.putObject(request);
|
167
|
+
}
|
168
|
+
|
169
|
+
private void closeCurrent()
|
170
|
+
{
|
171
|
+
if (current == null) {
|
172
|
+
return;
|
173
|
+
}
|
174
|
+
|
175
|
+
try {
|
176
|
+
putFile(tempFilePath, buildCurrentKey());
|
177
|
+
fileIndex++;
|
178
|
+
}
|
179
|
+
finally {
|
180
|
+
try {
|
181
|
+
current.close();
|
182
|
+
current = null;
|
183
|
+
}
|
184
|
+
catch (IOException e) {
|
185
|
+
throw new RuntimeException(e);
|
186
|
+
}
|
187
|
+
finally {
|
188
|
+
deleteTempFile();
|
189
|
+
}
|
190
|
+
}
|
191
|
+
}
|
192
|
+
|
193
|
+
@Override
|
194
|
+
public void nextFile()
|
195
|
+
{
|
196
|
+
closeCurrent();
|
197
|
+
|
198
|
+
try {
|
199
|
+
tempFilePath = newTempFile(tempPathPrefix);
|
200
|
+
|
201
|
+
log.info("Writing S3 file '{}'", buildCurrentKey());
|
202
|
+
|
203
|
+
current = Files.newOutputStream(tempFilePath);
|
204
|
+
}
|
205
|
+
catch (IOException e) {
|
206
|
+
throw new RuntimeException(e);
|
207
|
+
}
|
208
|
+
}
|
209
|
+
|
210
|
+
@Override
|
211
|
+
public void add(Buffer buffer)
|
212
|
+
{
|
213
|
+
if (current == null) {
|
214
|
+
throw new IllegalStateException(
|
215
|
+
"nextFile() must be called before poll()");
|
216
|
+
}
|
217
|
+
|
218
|
+
try {
|
219
|
+
current.write(buffer.array(), buffer.offset(), buffer.limit());
|
220
|
+
}
|
221
|
+
catch (IOException ex) {
|
222
|
+
throw new RuntimeException(ex);
|
223
|
+
}
|
224
|
+
finally {
|
225
|
+
buffer.release();
|
226
|
+
}
|
227
|
+
}
|
228
|
+
|
229
|
+
@Override
|
230
|
+
public void finish()
|
231
|
+
{
|
232
|
+
closeCurrent();
|
233
|
+
}
|
234
|
+
|
235
|
+
@Override
|
236
|
+
public void close()
|
237
|
+
{
|
238
|
+
closeCurrent();
|
239
|
+
}
|
240
|
+
|
241
|
+
@Override
|
242
|
+
public void abort()
|
243
|
+
{
|
244
|
+
deleteTempFile();
|
245
|
+
}
|
246
|
+
|
247
|
+
@Override
|
248
|
+
public TaskReport commit()
|
249
|
+
{
|
250
|
+
TaskReport report = Exec.newTaskReport();
|
251
|
+
return report;
|
252
|
+
}
|
253
|
+
}
|
254
|
+
|
255
|
+
private void validateSequenceFormat(PluginTask task)
|
256
|
+
{
|
257
|
+
try {
|
258
|
+
@SuppressWarnings("unused")
|
259
|
+
String dontCare = String.format(Locale.ENGLISH,
|
260
|
+
task.getSequenceFormat(), 0, 0);
|
261
|
+
}
|
262
|
+
catch (IllegalFormatException ex) {
|
263
|
+
throw new ConfigException(
|
264
|
+
"Invalid sequence_format: parameter for file output plugin",
|
265
|
+
ex);
|
266
|
+
}
|
267
|
+
}
|
268
|
+
|
269
|
+
@Override
|
270
|
+
public ConfigDiff transaction(ConfigSource config, int taskCount,
|
271
|
+
Control control)
|
272
|
+
{
|
273
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
274
|
+
|
275
|
+
validateSequenceFormat(task);
|
276
|
+
|
277
|
+
return resume(task.dump(), taskCount, control);
|
278
|
+
}
|
279
|
+
|
280
|
+
@Override
|
281
|
+
public ConfigDiff resume(TaskSource taskSource, int taskCount,
|
282
|
+
Control control)
|
283
|
+
{
|
284
|
+
control.run(taskSource);
|
285
|
+
return Exec.newConfigDiff();
|
286
|
+
}
|
287
|
+
|
288
|
+
@Override
|
289
|
+
public void cleanup(TaskSource taskSource, int taskCount,
|
290
|
+
List<TaskReport> successTaskReports)
|
291
|
+
{
|
292
|
+
}
|
293
|
+
|
294
|
+
@Override
|
295
|
+
public TransactionalFileOutput open(TaskSource taskSource, int taskIndex)
|
296
|
+
{
|
297
|
+
PluginTask task = taskSource.loadTask(PluginTask.class);
|
298
|
+
|
299
|
+
return new S3FileOutput(task, taskIndex);
|
300
|
+
}
|
301
|
+
}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-s3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Manabu Takayama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-03-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -49,11 +49,14 @@ files:
|
|
49
49
|
- LICENSE.txt
|
50
50
|
- README.md
|
51
51
|
- build.gradle
|
52
|
+
- config/checkstyle/checkstyle.xml
|
53
|
+
- config/checkstyle/default.xml
|
52
54
|
- gradle/wrapper/gradle-wrapper.jar
|
53
55
|
- gradle/wrapper/gradle-wrapper.properties
|
54
56
|
- gradlew
|
55
57
|
- gradlew.bat
|
56
58
|
- lib/embulk/output/s3.rb
|
59
|
+
- settings.gradle
|
57
60
|
- src/main/java/org/embulk/output/S3FileOutputPlugin.java
|
58
61
|
- src/test/java/org/embulk/output/TestS3FileOutputPlugin.java
|
59
62
|
- classpath/aws-java-sdk-core-1.10.26.jar
|
@@ -61,7 +64,7 @@ files:
|
|
61
64
|
- classpath/aws-java-sdk-s3-1.10.26.jar
|
62
65
|
- classpath/commons-codec-1.6.jar
|
63
66
|
- classpath/commons-logging-1.1.3.jar
|
64
|
-
- classpath/embulk-output-s3-1.
|
67
|
+
- classpath/embulk-output-s3-1.2.0.jar
|
65
68
|
- classpath/httpclient-4.3.6.jar
|
66
69
|
- classpath/httpcore-4.3.3.jar
|
67
70
|
homepage: https://github.com/llibra/embulk-output-s3
|