embulk-output-s3 1.1.0 → 1.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +13 -11
  3. data/README.md +60 -39
  4. data/build.gradle +77 -59
  5. data/config/checkstyle/checkstyle.xml +128 -0
  6. data/config/checkstyle/default.xml +108 -0
  7. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  8. data/gradle/wrapper/gradle-wrapper.properties +6 -6
  9. data/settings.gradle +1 -0
  10. data/src/main/java/org/embulk/output/S3FileOutputPlugin.java +301 -258
  11. metadata +6 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4d30bd3aaf8225192a198c2272068b535ff03ae2
4
- data.tar.gz: 67c729c619c8d5eb5366f6268719d9e559736be6
3
+ metadata.gz: 6494f870318029670e7e568288e0acfd63eb0ce0
4
+ data.tar.gz: fe7baf6a69629a714530a67d2aa2c718e6eb02ae
5
5
  SHA512:
6
- metadata.gz: f8c94a94091c02e2f38c649372b72fb3a37e7f0cb514addc201fe3efe76763757c633b5b9367b4448dde3ecc4f167336dea0458f2d08bb03cf0842e95dc2bb84
7
- data.tar.gz: df4d41d74fdd8c0c3bd8fbbb428471f0f99d61fdcaf6f645c6dee633f3359c3c75a7eaf79bdc2361404b5dde6029ffa3c23bd04a5a3533da871539fb8251ac7b
6
+ metadata.gz: c0f8a74ea56053b6daa9ffc981a52af413adc47ebd63f56fac609967125927e731e782294d5db93353dd5bf35fbb0b8e640cd76f6c6cc559aa83eec89017290c
7
+ data.tar.gz: f64f27ad7fd6b21d27843dcb46c486522064915abf42e9058d850248b590520010fcd1d4431f128f3bd30bddab6f44d00ce36b46075c13b533fd122923b58cd8
data/.gitignore CHANGED
@@ -1,11 +1,13 @@
1
- *~
2
- /pkg/
3
- /tmp/
4
- .gradle/
5
- /classpath/
6
- build/
7
- .idea
8
- .settings/
9
- bin/
10
- .classpath
11
- .project
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ .gradle/
5
+ /classpath/
6
+ build/
7
+ .idea
8
+ .settings/
9
+ bin/
10
+ .classpath
11
+ .project
12
+ *.gemspec
13
+ *.iml
data/README.md CHANGED
@@ -1,39 +1,60 @@
1
- # S3 file output plugin for Embulk
2
-
3
- ## Overview
4
-
5
- * **Plugin type**: file output
6
- * **Load all or nothing**: no
7
- * **Resume supported**: yes
8
- * **Cleanup supported**: yes
9
-
10
- ## Configuration
11
-
12
- - **path_prefix**: prefix of target keys (string, required)
13
- - **file_ext**: suffix of target keys (string, required)
14
- - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
- - **bucket**: S3 bucket name (string, required)
16
- - **endpoint**: S3 endpoint login user name (string, optional)
17
- - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
18
- - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
19
- - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
-
21
- ## Example
22
-
23
- ```yaml
24
- path_prefix: logs/out
25
- file_ext: .csv
26
- bucket: my-s3-bucket
27
- endpoint: s3-us-west-1.amazonaws.com
28
- access_key_id: ABCXYZ123ABCXYZ123
29
- secret_access_key: AbCxYz123aBcXyZ123
30
- formatter:
31
- type: csv
32
- ```
33
-
34
-
35
- ## Build
36
-
37
- ```
38
- $ ./gradlew gem
39
- ```
1
+ # S3 file output plugin for Embulk
2
+
3
+ ## Developers
4
+
5
+ * Manabu Takayama <learn.libra@gmail.com>
6
+ * toyama hiroshi <toyama0919@gmail.com>
7
+ * Civitaspo <civitaspo@gmail.com>
8
+
9
+ ## Overview
10
+
11
+ * **Plugin type**: file output
12
+ * **Load all or nothing**: no
13
+ * **Resume supported**: yes
14
+ * **Cleanup supported**: yes
15
+
16
+ ## Configuration
17
+
18
+ - **path_prefix**: prefix of target keys (string, required)
19
+ - **file_ext**: suffix of target keys (string, required)
20
+ - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
21
+ - **bucket**: S3 bucket name (string, required)
22
+ - **endpoint**: S3 endpoint login user name (string, optional)
23
+ - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
24
+ - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
25
+ - **tmp_path_prefix**: prefix of temporary files (string, default: 'embulk-output-s3-')
26
+ - **canned_acl**: canned access control list for created objects ([enum](#cannedaccesscontrollist), default: null)
27
+
28
+ ### CannedAccessControlList
29
+ you can choose one of the below list.
30
+
31
+ - AuthenticatedRead
32
+ - AwsExecRead
33
+ - BucketOwnerFullControl
34
+ - BucketOwnerRead
35
+ - LogDeliveryWrite
36
+ - Private
37
+ - PublicRead
38
+ - PublicReadWrite
39
+
40
+ cf. http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html
41
+
42
+ ## Example
43
+
44
+ ```yaml
45
+ path_prefix: logs/out
46
+ file_ext: .csv
47
+ bucket: my-s3-bucket
48
+ endpoint: s3-us-west-1.amazonaws.com
49
+ access_key_id: ABCXYZ123ABCXYZ123
50
+ secret_access_key: AbCxYz123aBcXyZ123
51
+ formatter:
52
+ type: csv
53
+ ```
54
+
55
+
56
+ ## Build
57
+
58
+ ```
59
+ $ ./gradlew gem
60
+ ```
data/build.gradle CHANGED
@@ -1,59 +1,77 @@
1
- plugins {
2
- id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
4
- id "java"
5
- }
6
- import com.github.jrubygradle.JRubyExec
7
- repositories {
8
- mavenCentral()
9
- jcenter()
10
- }
11
- configurations {
12
- provided
13
- }
14
-
15
- version = "1.1.0"
16
-
17
- sourceCompatibility = 1.7
18
- targetCompatibility = 1.7
19
-
20
- dependencies {
21
- compile "org.embulk:embulk-core:0.7.5"
22
- provided "org.embulk:embulk-core:0.7.5"
23
- compile "com.amazonaws:aws-java-sdk-s3:1.10.26"
24
- testCompile "junit:junit:4.+"
25
- }
26
-
27
- task classpath(type: Copy, dependsOn: ["jar"]) {
28
- doFirst { file("classpath").deleteDir() }
29
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
30
- into "classpath"
31
- }
32
- clean { delete 'classpath' }
33
-
34
- task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
35
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
36
- script "build/gemspec"
37
- doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
38
- }
39
-
40
- task gemspec << { file("build/gemspec").write($/
41
- Gem::Specification.new do |spec|
42
- spec.name = "${project.name}"
43
- spec.version = "${project.version}"
44
- spec.authors = ["Manabu Takayama"]
45
- spec.summary = %[S3 file output plugin for Embulk]
46
- spec.description = %[Stores files on S3.]
47
- spec.email = ["learn.libra@gmail.com"]
48
- spec.licenses = ["MIT"]
49
- spec.homepage = "https://github.com/llibra/embulk-output-s3"
50
-
51
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
52
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
53
- spec.require_paths = ["lib"]
54
-
55
- spec.add_development_dependency 'bundler', ['~> 1.0']
56
- spec.add_development_dependency 'rake', ['>= 10.0']
57
- end
58
- /$)
59
- }
1
+ plugins {
2
+ id "com.jfrog.bintray" version "1.1"
3
+ id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "java"
5
+ id "checkstyle"
6
+ }
7
+ import com.github.jrubygradle.JRubyExec
8
+ repositories {
9
+ mavenCentral()
10
+ jcenter()
11
+ }
12
+ configurations {
13
+ provided
14
+ }
15
+
16
+ version = "1.2.0"
17
+
18
+ sourceCompatibility = 1.7
19
+ targetCompatibility = 1.7
20
+
21
+ dependencies {
22
+ compile "org.embulk:embulk-core:0.8.1"
23
+ provided "org.embulk:embulk-core:0.8.1"
24
+ compile "com.amazonaws:aws-java-sdk-s3:1.10.26"
25
+ testCompile "junit:junit:4.+"
26
+ }
27
+
28
+ task classpath(type: Copy, dependsOn: ["jar"]) {
29
+ doFirst { file("classpath").deleteDir() }
30
+ from (configurations.runtime - configurations.provided + files(jar.archivePath))
31
+ into "classpath"
32
+ }
33
+ clean { delete 'classpath' }
34
+
35
+ checkstyle {
36
+ configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
37
+ toolVersion = '6.14.1'
38
+ }
39
+ checkstyleMain {
40
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
41
+ ignoreFailures = true
42
+ }
43
+ checkstyleTest {
44
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
45
+ ignoreFailures = true
46
+ }
47
+ task checkstyle(type: Checkstyle) {
48
+ classpath = sourceSets.main.output + sourceSets.test.output
49
+ source = sourceSets.main.allJava + sourceSets.test.allJava
50
+ }
51
+
52
+ task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
53
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
54
+ script "build/gemspec"
55
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
56
+ }
57
+
58
+ task gemspec << { file("build/gemspec").write($/
59
+ Gem::Specification.new do |spec|
60
+ spec.name = "${project.name}"
61
+ spec.version = "${project.version}"
62
+ spec.authors = ["Manabu Takayama"]
63
+ spec.summary = %[S3 file output plugin for Embulk]
64
+ spec.description = %[Stores files on S3.]
65
+ spec.email = ["learn.libra@gmail.com"]
66
+ spec.licenses = ["MIT"]
67
+ spec.homepage = "https://github.com/llibra/embulk-output-s3"
68
+
69
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
70
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
71
+ spec.require_paths = ["lib"]
72
+
73
+ spec.add_development_dependency 'bundler', ['~> 1.0']
74
+ spec.add_development_dependency 'rake', ['>= 10.0']
75
+ end
76
+ /$)
77
+ }
data/config/checkstyle/checkstyle.xml ADDED
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
data/config/checkstyle/default.xml ADDED
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>
data/gradle/wrapper/gradle-wrapper.jar CHANGED
Binary file
data/gradle/wrapper/gradle-wrapper.properties CHANGED
@@ -1,6 +1,6 @@
1
- #Tue Aug 11 00:26:20 PDT 2015
2
- distributionBase=GRADLE_USER_HOME
3
- distributionPath=wrapper/dists
4
- zipStoreBase=GRADLE_USER_HOME
5
- zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
1
+ #Wed Jan 13 12:41:02 JST 2016
2
+ distributionBase=GRADLE_USER_HOME
3
+ distributionPath=wrapper/dists
4
+ zipStoreBase=GRADLE_USER_HOME
5
+ zipStorePath=wrapper/dists
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
data/settings.gradle ADDED
@@ -0,0 +1 @@
1
+ rootProject.name = 'embulk-output-s3'
data/src/main/java/org/embulk/output/S3FileOutputPlugin.java CHANGED
@@ -1,258 +1,301 @@
1
- package org.embulk.output;
2
-
3
- import java.io.IOException;
4
- import java.io.OutputStream;
5
- import java.nio.file.Files;
6
- import java.nio.file.Path;
7
- import java.util.IllegalFormatException;
8
- import java.util.List;
9
- import java.util.Locale;
10
-
11
- import org.embulk.config.TaskReport;
12
- import org.embulk.config.Config;
13
- import org.embulk.config.ConfigDefault;
14
- import org.embulk.config.ConfigDiff;
15
- import org.embulk.config.ConfigException;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskSource;
19
- import org.embulk.spi.Buffer;
20
- import org.embulk.spi.Exec;
21
- import org.embulk.spi.FileOutput;
22
- import org.embulk.spi.FileOutputPlugin;
23
- import org.embulk.spi.TransactionalFileOutput;
24
- import org.slf4j.Logger;
25
-
26
- import com.amazonaws.ClientConfiguration;
27
- import com.amazonaws.auth.BasicAWSCredentials;
28
- import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
29
- import com.amazonaws.services.s3.AmazonS3Client;
30
- import com.amazonaws.services.s3.model.PutObjectRequest;
31
- import com.google.common.base.Optional;
32
-
33
- public class S3FileOutputPlugin implements FileOutputPlugin {
34
- public interface PluginTask extends Task {
35
- @Config("path_prefix")
36
- public String getPathPrefix();
37
-
38
- @Config("file_ext")
39
- public String getFileNameExtension();
40
-
41
- @Config("sequence_format")
42
- @ConfigDefault("\".%03d.%02d\"")
43
- public String getSequenceFormat();
44
-
45
- @Config("bucket")
46
- public String getBucket();
47
-
48
- @Config("endpoint")
49
- @ConfigDefault("null")
50
- public Optional<String> getEndpoint();
51
-
52
- @Config("access_key_id")
53
- @ConfigDefault("null")
54
- public Optional<String> getAccessKeyId();
55
-
56
- @Config("secret_access_key")
57
- @ConfigDefault("null")
58
- public Optional<String> getSecretAccessKey();
59
-
60
- @Config("tmp_path_prefix")
61
- @ConfigDefault("\"embulk-output-s3-\"")
62
- public String getTempPathPrefix();
63
- }
64
-
65
- public static class S3FileOutput implements FileOutput,
66
- TransactionalFileOutput {
67
- private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
68
-
69
- private final String bucket;
70
- private final String pathPrefix;
71
- private final String sequenceFormat;
72
- private final String fileNameExtension;
73
- private final String tempPathPrefix;
74
-
75
- private int taskIndex;
76
- private int fileIndex;
77
- private AmazonS3Client client;
78
- private OutputStream current;
79
- private Path tempFilePath;
80
-
81
- private static AmazonS3Client newS3Client(PluginTask task) {
82
- AmazonS3Client client = null;
83
-
84
- if (task.getAccessKeyId().isPresent()) {
85
- BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
86
- task.getAccessKeyId().get(), task.getSecretAccessKey().get());
87
-
88
- ClientConfiguration config = new ClientConfiguration();
89
- // TODO: Support more configurations.
90
-
91
- client = new AmazonS3Client(basicAWSCredentials, config);
92
- } else {
93
- if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
94
- client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
95
- } else { // IAM ROLE
96
- client = new AmazonS3Client();
97
- }
98
- }
99
- if (task.getEndpoint().isPresent()) {
100
- client.setEndpoint(task.getEndpoint().get());
101
- }
102
-
103
- return client;
104
- }
105
-
106
- public S3FileOutput(PluginTask task, int taskIndex) {
107
- this.taskIndex = taskIndex;
108
- this.client = newS3Client(task);
109
- this.bucket = task.getBucket();
110
- this.pathPrefix = task.getPathPrefix();
111
- this.sequenceFormat = task.getSequenceFormat();
112
- this.fileNameExtension = task.getFileNameExtension();
113
- this.tempPathPrefix = task.getTempPathPrefix();
114
- }
115
-
116
- private static Path newTempFile(String prefix) throws IOException {
117
- return Files.createTempFile(prefix, null);
118
- }
119
-
120
- private void deleteTempFile() {
121
- if (tempFilePath == null) {
122
- return;
123
- }
124
-
125
- try {
126
- Files.delete(tempFilePath);
127
- tempFilePath = null;
128
- } catch (IOException e) {
129
- throw new RuntimeException(e);
130
- }
131
- }
132
-
133
- private String buildCurrentKey() {
134
- String sequence = String.format(sequenceFormat, taskIndex,
135
- fileIndex);
136
- return pathPrefix + sequence + fileNameExtension;
137
- }
138
-
139
- private void putFile(Path from, String key) {
140
- PutObjectRequest request = new PutObjectRequest(bucket, key,
141
- from.toFile());
142
- client.putObject(request);
143
- }
144
-
145
- private void closeCurrent() {
146
- if (current == null) {
147
- return;
148
- }
149
-
150
- try {
151
- putFile(tempFilePath, buildCurrentKey());
152
- fileIndex++;
153
- } finally {
154
- try {
155
- current.close();
156
- current = null;
157
- } catch (IOException e) {
158
- throw new RuntimeException(e);
159
- } finally {
160
- deleteTempFile();
161
- }
162
- }
163
- }
164
-
165
- @Override
166
- public void nextFile() {
167
- closeCurrent();
168
-
169
- try {
170
- tempFilePath = newTempFile(tempPathPrefix);
171
-
172
- log.info("Writing S3 file '{}'", buildCurrentKey());
173
-
174
- current = Files.newOutputStream(tempFilePath);
175
- } catch (IOException e) {
176
- throw new RuntimeException(e);
177
- }
178
- }
179
-
180
- @Override
181
- public void add(Buffer buffer) {
182
- if (current == null) {
183
- throw new IllegalStateException(
184
- "nextFile() must be called before poll()");
185
- }
186
-
187
- try {
188
- current.write(buffer.array(), buffer.offset(), buffer.limit());
189
- } catch (IOException ex) {
190
- throw new RuntimeException(ex);
191
- } finally {
192
- buffer.release();
193
- }
194
- }
195
-
196
- @Override
197
- public void finish() {
198
- closeCurrent();
199
- }
200
-
201
- @Override
202
- public void close() {
203
- closeCurrent();
204
- }
205
-
206
- @Override
207
- public void abort() {
208
- deleteTempFile();
209
- }
210
-
211
- @Override
212
- public TaskReport commit() {
213
- TaskReport report = Exec.newTaskReport();
214
- return report;
215
- }
216
- }
217
-
218
- private void validateSequenceFormat(PluginTask task) {
219
- try {
220
- @SuppressWarnings("unused")
221
- String dontCare = String.format(Locale.ENGLISH,
222
- task.getSequenceFormat(), 0, 0);
223
- } catch (IllegalFormatException ex) {
224
- throw new ConfigException(
225
- "Invalid sequence_format: parameter for file output plugin",
226
- ex);
227
- }
228
- }
229
-
230
- @Override
231
- public ConfigDiff transaction(ConfigSource config, int taskCount,
232
- Control control) {
233
- PluginTask task = config.loadConfig(PluginTask.class);
234
-
235
- validateSequenceFormat(task);
236
-
237
- return resume(task.dump(), taskCount, control);
238
- }
239
-
240
- @Override
241
- public ConfigDiff resume(TaskSource taskSource, int taskCount,
242
- Control control) {
243
- control.run(taskSource);
244
- return Exec.newConfigDiff();
245
- }
246
-
247
- @Override
248
- public void cleanup(TaskSource taskSource, int taskCount,
249
- List<TaskReport> successTaskReports) {
250
- }
251
-
252
- @Override
253
- public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
254
- PluginTask task = taskSource.loadTask(PluginTask.class);
255
-
256
- return new S3FileOutput(task, taskIndex);
257
- }
258
- }
1
+ package org.embulk.output;
2
+
3
+ import com.amazonaws.ClientConfiguration;
4
+ import com.amazonaws.auth.BasicAWSCredentials;
5
+ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
6
+ import com.amazonaws.services.s3.AmazonS3Client;
7
+ import com.amazonaws.services.s3.model.CannedAccessControlList;
8
+ import com.amazonaws.services.s3.model.PutObjectRequest;
9
+ import com.google.common.base.Optional;
10
+ import org.embulk.config.Config;
11
+ import org.embulk.config.ConfigDefault;
12
+ import org.embulk.config.ConfigDiff;
13
+ import org.embulk.config.ConfigException;
14
+ import org.embulk.config.ConfigSource;
15
+ import org.embulk.config.Task;
16
+ import org.embulk.config.TaskReport;
17
+ import org.embulk.config.TaskSource;
18
+ import org.embulk.spi.Buffer;
19
+ import org.embulk.spi.Exec;
20
+ import org.embulk.spi.FileOutput;
21
+ import org.embulk.spi.FileOutputPlugin;
22
+ import org.embulk.spi.TransactionalFileOutput;
23
+ import org.slf4j.Logger;
24
+
25
+ import java.io.IOException;
26
+ import java.io.OutputStream;
27
+ import java.nio.file.Files;
28
+ import java.nio.file.Path;
29
+ import java.util.IllegalFormatException;
30
+ import java.util.List;
31
+ import java.util.Locale;
32
+
33
+ public class S3FileOutputPlugin
34
+ implements FileOutputPlugin
35
+ {
36
+ public interface PluginTask
37
+ extends Task
38
+ {
39
+ @Config("path_prefix")
40
+ String getPathPrefix();
41
+
42
+ @Config("file_ext")
43
+ String getFileNameExtension();
44
+
45
+ @Config("sequence_format")
46
+ @ConfigDefault("\".%03d.%02d\"")
47
+ String getSequenceFormat();
48
+
49
+ @Config("bucket")
50
+ String getBucket();
51
+
52
+ @Config("endpoint")
53
+ @ConfigDefault("null")
54
+ Optional<String> getEndpoint();
55
+
56
+ @Config("access_key_id")
57
+ @ConfigDefault("null")
58
+ Optional<String> getAccessKeyId();
59
+
60
+ @Config("secret_access_key")
61
+ @ConfigDefault("null")
62
+ Optional<String> getSecretAccessKey();
63
+
64
+ @Config("tmp_path_prefix")
65
+ @ConfigDefault("\"embulk-output-s3-\"")
66
+ String getTempPathPrefix();
67
+
68
+ @Config("canned_acl")
69
+ @ConfigDefault("null")
70
+ Optional<CannedAccessControlList> getCannedAccessControlList();
71
+ }
72
+
73
+ public static class S3FileOutput
74
+ implements FileOutput,
75
+ TransactionalFileOutput
76
+ {
77
+ private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
78
+
79
+ private final String bucket;
80
+ private final String pathPrefix;
81
+ private final String sequenceFormat;
82
+ private final String fileNameExtension;
83
+ private final String tempPathPrefix;
84
+ private final Optional<CannedAccessControlList> cannedAccessControlListOptional;
85
+
86
+ private int taskIndex;
87
+ private int fileIndex;
88
+ private AmazonS3Client client;
89
+ private OutputStream current;
90
+ private Path tempFilePath;
91
+
92
+ private static AmazonS3Client newS3Client(PluginTask task)
93
+ {
94
+ AmazonS3Client client;
95
+
96
+ if (task.getAccessKeyId().isPresent()) {
97
+ BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
98
+ task.getAccessKeyId().get(), task.getSecretAccessKey().get());
99
+
100
+ ClientConfiguration config = new ClientConfiguration();
101
+ // TODO: Support more configurations.
102
+
103
+ client = new AmazonS3Client(basicAWSCredentials, config);
104
+ }
105
+ else {
106
+ if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
107
+ client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
108
+ }
109
+ else { // IAM ROLE
110
+ client = new AmazonS3Client();
111
+ }
112
+ }
113
+ if (task.getEndpoint().isPresent()) {
114
+ client.setEndpoint(task.getEndpoint().get());
115
+ }
116
+
117
+ return client;
118
+ }
119
+
120
+ public S3FileOutput(PluginTask task, int taskIndex)
121
+ {
122
+ this.taskIndex = taskIndex;
123
+ this.client = newS3Client(task);
124
+ this.bucket = task.getBucket();
125
+ this.pathPrefix = task.getPathPrefix();
126
+ this.sequenceFormat = task.getSequenceFormat();
127
+ this.fileNameExtension = task.getFileNameExtension();
128
+ this.tempPathPrefix = task.getTempPathPrefix();
129
+ this.cannedAccessControlListOptional = task.getCannedAccessControlList();
130
+ }
131
+
132
+ private static Path newTempFile(String prefix)
133
+ throws IOException
134
+ {
135
+ return Files.createTempFile(prefix, null);
136
+ }
137
+
138
+ private void deleteTempFile()
139
+ {
140
+ if (tempFilePath == null) {
141
+ return;
142
+ }
143
+
144
+ try {
145
+ Files.delete(tempFilePath);
146
+ tempFilePath = null;
147
+ }
148
+ catch (IOException e) {
149
+ throw new RuntimeException(e);
150
+ }
151
+ }
152
+
153
+ private String buildCurrentKey()
154
+ {
155
+ String sequence = String.format(sequenceFormat, taskIndex,
156
+ fileIndex);
157
+ return pathPrefix + sequence + fileNameExtension;
158
+ }
159
+
160
+ private void putFile(Path from, String key)
161
+ {
162
+ PutObjectRequest request = new PutObjectRequest(bucket, key, from.toFile());
163
+ if (cannedAccessControlListOptional.isPresent()) {
164
+ request.withCannedAcl(cannedAccessControlListOptional.get());
165
+ }
166
+ client.putObject(request);
167
+ }
168
+
169
+ private void closeCurrent()
170
+ {
171
+ if (current == null) {
172
+ return;
173
+ }
174
+
175
+ try {
176
+ putFile(tempFilePath, buildCurrentKey());
177
+ fileIndex++;
178
+ }
179
+ finally {
180
+ try {
181
+ current.close();
182
+ current = null;
183
+ }
184
+ catch (IOException e) {
185
+ throw new RuntimeException(e);
186
+ }
187
+ finally {
188
+ deleteTempFile();
189
+ }
190
+ }
191
+ }
192
+
193
+ @Override
194
+ public void nextFile()
195
+ {
196
+ closeCurrent();
197
+
198
+ try {
199
+ tempFilePath = newTempFile(tempPathPrefix);
200
+
201
+ log.info("Writing S3 file '{}'", buildCurrentKey());
202
+
203
+ current = Files.newOutputStream(tempFilePath);
204
+ }
205
+ catch (IOException e) {
206
+ throw new RuntimeException(e);
207
+ }
208
+ }
209
+
210
+ @Override
211
+ public void add(Buffer buffer)
212
+ {
213
+ if (current == null) {
214
+ throw new IllegalStateException(
215
+ "nextFile() must be called before poll()");
216
+ }
217
+
218
+ try {
219
+ current.write(buffer.array(), buffer.offset(), buffer.limit());
220
+ }
221
+ catch (IOException ex) {
222
+ throw new RuntimeException(ex);
223
+ }
224
+ finally {
225
+ buffer.release();
226
+ }
227
+ }
228
+
229
+ @Override
230
+ public void finish()
231
+ {
232
+ closeCurrent();
233
+ }
234
+
235
+ @Override
236
+ public void close()
237
+ {
238
+ closeCurrent();
239
+ }
240
+
241
+ @Override
242
+ public void abort()
243
+ {
244
+ deleteTempFile();
245
+ }
246
+
247
+ @Override
248
+ public TaskReport commit()
249
+ {
250
+ TaskReport report = Exec.newTaskReport();
251
+ return report;
252
+ }
253
+ }
254
+
255
+ private void validateSequenceFormat(PluginTask task)
256
+ {
257
+ try {
258
+ @SuppressWarnings("unused")
259
+ String dontCare = String.format(Locale.ENGLISH,
260
+ task.getSequenceFormat(), 0, 0);
261
+ }
262
+ catch (IllegalFormatException ex) {
263
+ throw new ConfigException(
264
+ "Invalid sequence_format: parameter for file output plugin",
265
+ ex);
266
+ }
267
+ }
268
+
269
+ @Override
270
+ public ConfigDiff transaction(ConfigSource config, int taskCount,
271
+ Control control)
272
+ {
273
+ PluginTask task = config.loadConfig(PluginTask.class);
274
+
275
+ validateSequenceFormat(task);
276
+
277
+ return resume(task.dump(), taskCount, control);
278
+ }
279
+
280
+ @Override
281
+ public ConfigDiff resume(TaskSource taskSource, int taskCount,
282
+ Control control)
283
+ {
284
+ control.run(taskSource);
285
+ return Exec.newConfigDiff();
286
+ }
287
+
288
+ @Override
289
+ public void cleanup(TaskSource taskSource, int taskCount,
290
+ List<TaskReport> successTaskReports)
291
+ {
292
+ }
293
+
294
+ @Override
295
+ public TransactionalFileOutput open(TaskSource taskSource, int taskIndex)
296
+ {
297
+ PluginTask task = taskSource.loadTask(PluginTask.class);
298
+
299
+ return new S3FileOutput(task, taskIndex);
300
+ }
301
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Takayama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-28 00:00:00.000000000 Z
11
+ date: 2016-03-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,11 +49,14 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
+ - config/checkstyle/checkstyle.xml
53
+ - config/checkstyle/default.xml
52
54
  - gradle/wrapper/gradle-wrapper.jar
53
55
  - gradle/wrapper/gradle-wrapper.properties
54
56
  - gradlew
55
57
  - gradlew.bat
56
58
  - lib/embulk/output/s3.rb
59
+ - settings.gradle
57
60
  - src/main/java/org/embulk/output/S3FileOutputPlugin.java
58
61
  - src/test/java/org/embulk/output/TestS3FileOutputPlugin.java
59
62
  - classpath/aws-java-sdk-core-1.10.26.jar
@@ -61,7 +64,7 @@ files:
61
64
  - classpath/aws-java-sdk-s3-1.10.26.jar
62
65
  - classpath/commons-codec-1.6.jar
63
66
  - classpath/commons-logging-1.1.3.jar
64
- - classpath/embulk-output-s3-1.1.0.jar
67
+ - classpath/embulk-output-s3-1.2.0.jar
65
68
  - classpath/httpclient-4.3.6.jar
66
69
  - classpath/httpcore-4.3.3.jar
67
70
  homepage: https://github.com/llibra/embulk-output-s3