embulk-output-s3 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (11) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +13 -11
  3. data/README.md +60 -39
  4. data/build.gradle +77 -59
  5. data/config/checkstyle/checkstyle.xml +128 -0
  6. data/config/checkstyle/default.xml +108 -0
  7. data/gradle/wrapper/gradle-wrapper.jar +0 -0
  8. data/gradle/wrapper/gradle-wrapper.properties +6 -6
  9. data/settings.gradle +1 -0
  10. data/src/main/java/org/embulk/output/S3FileOutputPlugin.java +301 -258
  11. metadata +6 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 4d30bd3aaf8225192a198c2272068b535ff03ae2
4
- data.tar.gz: 67c729c619c8d5eb5366f6268719d9e559736be6
3
+ metadata.gz: 6494f870318029670e7e568288e0acfd63eb0ce0
4
+ data.tar.gz: fe7baf6a69629a714530a67d2aa2c718e6eb02ae
5
5
  SHA512:
6
- metadata.gz: f8c94a94091c02e2f38c649372b72fb3a37e7f0cb514addc201fe3efe76763757c633b5b9367b4448dde3ecc4f167336dea0458f2d08bb03cf0842e95dc2bb84
7
- data.tar.gz: df4d41d74fdd8c0c3bd8fbbb428471f0f99d61fdcaf6f645c6dee633f3359c3c75a7eaf79bdc2361404b5dde6029ffa3c23bd04a5a3533da871539fb8251ac7b
6
+ metadata.gz: c0f8a74ea56053b6daa9ffc981a52af413adc47ebd63f56fac609967125927e731e782294d5db93353dd5bf35fbb0b8e640cd76f6c6cc559aa83eec89017290c
7
+ data.tar.gz: f64f27ad7fd6b21d27843dcb46c486522064915abf42e9058d850248b590520010fcd1d4431f128f3bd30bddab6f44d00ce36b46075c13b533fd122923b58cd8
data/.gitignore CHANGED
@@ -1,11 +1,13 @@
1
- *~
2
- /pkg/
3
- /tmp/
4
- .gradle/
5
- /classpath/
6
- build/
7
- .idea
8
- .settings/
9
- bin/
10
- .classpath
11
- .project
1
+ *~
2
+ /pkg/
3
+ /tmp/
4
+ .gradle/
5
+ /classpath/
6
+ build/
7
+ .idea
8
+ .settings/
9
+ bin/
10
+ .classpath
11
+ .project
12
+ *.gemspec
13
+ *.iml
data/README.md CHANGED
@@ -1,39 +1,60 @@
1
- # S3 file output plugin for Embulk
2
-
3
- ## Overview
4
-
5
- * **Plugin type**: file output
6
- * **Load all or nothing**: no
7
- * **Resume supported**: yes
8
- * **Cleanup supported**: yes
9
-
10
- ## Configuration
11
-
12
- - **path_prefix**: prefix of target keys (string, required)
13
- - **file_ext**: suffix of target keys (string, required)
14
- - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
15
- - **bucket**: S3 bucket name (string, required)
16
- - **endpoint**: S3 endpoint login user name (string, optional)
17
- - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
18
- - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
19
- - **tmp_path_prefix**: prefix of temporary files (string, defualt: 'embulk-output-s3-')
20
-
21
- ## Example
22
-
23
- ```yaml
24
- path_prefix: logs/out
25
- file_ext: .csv
26
- bucket: my-s3-bucket
27
- endpoint: s3-us-west-1.amazonaws.com
28
- access_key_id: ABCXYZ123ABCXYZ123
29
- secret_access_key: AbCxYz123aBcXyZ123
30
- formatter:
31
- type: csv
32
- ```
33
-
34
-
35
- ## Build
36
-
37
- ```
38
- $ ./gradlew gem
39
- ```
1
+ # S3 file output plugin for Embulk
2
+
3
+ ## Developers
4
+
5
+ * Manabu Takayama <learn.libra@gmail.com>
6
+ * toyama hiroshi <toyama0919@gmail.com>
7
+ * Civitaspo <civitaspo@gmail.com>
8
+
9
+ ## Overview
10
+
11
+ * **Plugin type**: file output
12
+ * **Load all or nothing**: no
13
+ * **Resume supported**: yes
14
+ * **Cleanup supported**: yes
15
+
16
+ ## Configuration
17
+
18
+ - **path_prefix**: prefix of target keys (string, required)
19
+ - **file_ext**: suffix of target keys (string, required)
20
+ - **sequence_format**: format for sequence part of target keys (string, default: '.%03d.%02d')
21
+ - **bucket**: S3 bucket name (string, required)
22
+ - **endpoint**: S3 endpoint login user name (string, optional)
23
+ - **access_key_id**: AWS access key id. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
24
+ - **secret_access_key**: AWS secret key. This parameter is required when your agent is not running on EC2 instance with an IAM Role. (string, defualt: null)
25
+ - **tmp_path_prefix**: prefix of temporary files (string, default: 'embulk-output-s3-')
26
+ - **canned_acl**: canned access control list for created objects ([enum](#cannedaccesscontrollist), default: null)
27
+
28
+ ### CannedAccessControlList
29
+ you can choose one of the below list.
30
+
31
+ - AuthenticatedRead
32
+ - AwsExecRead
33
+ - BucketOwnerFullControl
34
+ - BucketOwnerRead
35
+ - LogDeliveryWrite
36
+ - Private
37
+ - PublicRead
38
+ - PublicReadWrite
39
+
40
+ cf. http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/model/CannedAccessControlList.html
41
+
42
+ ## Example
43
+
44
+ ```yaml
45
+ path_prefix: logs/out
46
+ file_ext: .csv
47
+ bucket: my-s3-bucket
48
+ endpoint: s3-us-west-1.amazonaws.com
49
+ access_key_id: ABCXYZ123ABCXYZ123
50
+ secret_access_key: AbCxYz123aBcXyZ123
51
+ formatter:
52
+ type: csv
53
+ ```
54
+
55
+
56
+ ## Build
57
+
58
+ ```
59
+ $ ./gradlew gem
60
+ ```
data/build.gradle CHANGED
@@ -1,59 +1,77 @@
1
- plugins {
2
- id "com.jfrog.bintray" version "1.1"
3
- id "com.github.jruby-gradle.base" version "0.1.5"
4
- id "java"
5
- }
6
- import com.github.jrubygradle.JRubyExec
7
- repositories {
8
- mavenCentral()
9
- jcenter()
10
- }
11
- configurations {
12
- provided
13
- }
14
-
15
- version = "1.1.0"
16
-
17
- sourceCompatibility = 1.7
18
- targetCompatibility = 1.7
19
-
20
- dependencies {
21
- compile "org.embulk:embulk-core:0.7.5"
22
- provided "org.embulk:embulk-core:0.7.5"
23
- compile "com.amazonaws:aws-java-sdk-s3:1.10.26"
24
- testCompile "junit:junit:4.+"
25
- }
26
-
27
- task classpath(type: Copy, dependsOn: ["jar"]) {
28
- doFirst { file("classpath").deleteDir() }
29
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
30
- into "classpath"
31
- }
32
- clean { delete 'classpath' }
33
-
34
- task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
35
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
36
- script "build/gemspec"
37
- doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
38
- }
39
-
40
- task gemspec << { file("build/gemspec").write($/
41
- Gem::Specification.new do |spec|
42
- spec.name = "${project.name}"
43
- spec.version = "${project.version}"
44
- spec.authors = ["Manabu Takayama"]
45
- spec.summary = %[S3 file output plugin for Embulk]
46
- spec.description = %[Stores files on S3.]
47
- spec.email = ["learn.libra@gmail.com"]
48
- spec.licenses = ["MIT"]
49
- spec.homepage = "https://github.com/llibra/embulk-output-s3"
50
-
51
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
52
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
53
- spec.require_paths = ["lib"]
54
-
55
- spec.add_development_dependency 'bundler', ['~> 1.0']
56
- spec.add_development_dependency 'rake', ['>= 10.0']
57
- end
58
- /$)
59
- }
1
+ plugins {
2
+ id "com.jfrog.bintray" version "1.1"
3
+ id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "java"
5
+ id "checkstyle"
6
+ }
7
+ import com.github.jrubygradle.JRubyExec
8
+ repositories {
9
+ mavenCentral()
10
+ jcenter()
11
+ }
12
+ configurations {
13
+ provided
14
+ }
15
+
16
+ version = "1.2.0"
17
+
18
+ sourceCompatibility = 1.7
19
+ targetCompatibility = 1.7
20
+
21
+ dependencies {
22
+ compile "org.embulk:embulk-core:0.8.1"
23
+ provided "org.embulk:embulk-core:0.8.1"
24
+ compile "com.amazonaws:aws-java-sdk-s3:1.10.26"
25
+ testCompile "junit:junit:4.+"
26
+ }
27
+
28
+ task classpath(type: Copy, dependsOn: ["jar"]) {
29
+ doFirst { file("classpath").deleteDir() }
30
+ from (configurations.runtime - configurations.provided + files(jar.archivePath))
31
+ into "classpath"
32
+ }
33
+ clean { delete 'classpath' }
34
+
35
+ checkstyle {
36
+ configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
37
+ toolVersion = '6.14.1'
38
+ }
39
+ checkstyleMain {
40
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
41
+ ignoreFailures = true
42
+ }
43
+ checkstyleTest {
44
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
45
+ ignoreFailures = true
46
+ }
47
+ task checkstyle(type: Checkstyle) {
48
+ classpath = sourceSets.main.output + sourceSets.test.output
49
+ source = sourceSets.main.allJava + sourceSets.test.allJava
50
+ }
51
+
52
+ task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
53
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
54
+ script "build/gemspec"
55
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
56
+ }
57
+
58
+ task gemspec << { file("build/gemspec").write($/
59
+ Gem::Specification.new do |spec|
60
+ spec.name = "${project.name}"
61
+ spec.version = "${project.version}"
62
+ spec.authors = ["Manabu Takayama"]
63
+ spec.summary = %[S3 file output plugin for Embulk]
64
+ spec.description = %[Stores files on S3.]
65
+ spec.email = ["learn.libra@gmail.com"]
66
+ spec.licenses = ["MIT"]
67
+ spec.homepage = "https://github.com/llibra/embulk-output-s3"
68
+
69
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
70
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
71
+ spec.require_paths = ["lib"]
72
+
73
+ spec.add_development_dependency 'bundler', ['~> 1.0']
74
+ spec.add_development_dependency 'rake', ['>= 10.0']
75
+ end
76
+ /$)
77
+ }
data/config/checkstyle/checkstyle.xml ADDED
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
data/config/checkstyle/default.xml ADDED
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>
data/gradle/wrapper/gradle-wrapper.jar CHANGED
Binary file
data/gradle/wrapper/gradle-wrapper.properties CHANGED
@@ -1,6 +1,6 @@
1
- #Tue Aug 11 00:26:20 PDT 2015
2
- distributionBase=GRADLE_USER_HOME
3
- distributionPath=wrapper/dists
4
- zipStoreBase=GRADLE_USER_HOME
5
- zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip
1
+ #Wed Jan 13 12:41:02 JST 2016
2
+ distributionBase=GRADLE_USER_HOME
3
+ distributionPath=wrapper/dists
4
+ zipStoreBase=GRADLE_USER_HOME
5
+ zipStorePath=wrapper/dists
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
data/settings.gradle ADDED
@@ -0,0 +1 @@
1
+ rootProject.name = 'embulk-output-s3'
data/src/main/java/org/embulk/output/S3FileOutputPlugin.java CHANGED
@@ -1,258 +1,301 @@
1
- package org.embulk.output;
2
-
3
- import java.io.IOException;
4
- import java.io.OutputStream;
5
- import java.nio.file.Files;
6
- import java.nio.file.Path;
7
- import java.util.IllegalFormatException;
8
- import java.util.List;
9
- import java.util.Locale;
10
-
11
- import org.embulk.config.TaskReport;
12
- import org.embulk.config.Config;
13
- import org.embulk.config.ConfigDefault;
14
- import org.embulk.config.ConfigDiff;
15
- import org.embulk.config.ConfigException;
16
- import org.embulk.config.ConfigSource;
17
- import org.embulk.config.Task;
18
- import org.embulk.config.TaskSource;
19
- import org.embulk.spi.Buffer;
20
- import org.embulk.spi.Exec;
21
- import org.embulk.spi.FileOutput;
22
- import org.embulk.spi.FileOutputPlugin;
23
- import org.embulk.spi.TransactionalFileOutput;
24
- import org.slf4j.Logger;
25
-
26
- import com.amazonaws.ClientConfiguration;
27
- import com.amazonaws.auth.BasicAWSCredentials;
28
- import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
29
- import com.amazonaws.services.s3.AmazonS3Client;
30
- import com.amazonaws.services.s3.model.PutObjectRequest;
31
- import com.google.common.base.Optional;
32
-
33
- public class S3FileOutputPlugin implements FileOutputPlugin {
34
- public interface PluginTask extends Task {
35
- @Config("path_prefix")
36
- public String getPathPrefix();
37
-
38
- @Config("file_ext")
39
- public String getFileNameExtension();
40
-
41
- @Config("sequence_format")
42
- @ConfigDefault("\".%03d.%02d\"")
43
- public String getSequenceFormat();
44
-
45
- @Config("bucket")
46
- public String getBucket();
47
-
48
- @Config("endpoint")
49
- @ConfigDefault("null")
50
- public Optional<String> getEndpoint();
51
-
52
- @Config("access_key_id")
53
- @ConfigDefault("null")
54
- public Optional<String> getAccessKeyId();
55
-
56
- @Config("secret_access_key")
57
- @ConfigDefault("null")
58
- public Optional<String> getSecretAccessKey();
59
-
60
- @Config("tmp_path_prefix")
61
- @ConfigDefault("\"embulk-output-s3-\"")
62
- public String getTempPathPrefix();
63
- }
64
-
65
- public static class S3FileOutput implements FileOutput,
66
- TransactionalFileOutput {
67
- private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
68
-
69
- private final String bucket;
70
- private final String pathPrefix;
71
- private final String sequenceFormat;
72
- private final String fileNameExtension;
73
- private final String tempPathPrefix;
74
-
75
- private int taskIndex;
76
- private int fileIndex;
77
- private AmazonS3Client client;
78
- private OutputStream current;
79
- private Path tempFilePath;
80
-
81
- private static AmazonS3Client newS3Client(PluginTask task) {
82
- AmazonS3Client client = null;
83
-
84
- if (task.getAccessKeyId().isPresent()) {
85
- BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
86
- task.getAccessKeyId().get(), task.getSecretAccessKey().get());
87
-
88
- ClientConfiguration config = new ClientConfiguration();
89
- // TODO: Support more configurations.
90
-
91
- client = new AmazonS3Client(basicAWSCredentials, config);
92
- } else {
93
- if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
94
- client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
95
- } else { // IAM ROLE
96
- client = new AmazonS3Client();
97
- }
98
- }
99
- if (task.getEndpoint().isPresent()) {
100
- client.setEndpoint(task.getEndpoint().get());
101
- }
102
-
103
- return client;
104
- }
105
-
106
- public S3FileOutput(PluginTask task, int taskIndex) {
107
- this.taskIndex = taskIndex;
108
- this.client = newS3Client(task);
109
- this.bucket = task.getBucket();
110
- this.pathPrefix = task.getPathPrefix();
111
- this.sequenceFormat = task.getSequenceFormat();
112
- this.fileNameExtension = task.getFileNameExtension();
113
- this.tempPathPrefix = task.getTempPathPrefix();
114
- }
115
-
116
- private static Path newTempFile(String prefix) throws IOException {
117
- return Files.createTempFile(prefix, null);
118
- }
119
-
120
- private void deleteTempFile() {
121
- if (tempFilePath == null) {
122
- return;
123
- }
124
-
125
- try {
126
- Files.delete(tempFilePath);
127
- tempFilePath = null;
128
- } catch (IOException e) {
129
- throw new RuntimeException(e);
130
- }
131
- }
132
-
133
- private String buildCurrentKey() {
134
- String sequence = String.format(sequenceFormat, taskIndex,
135
- fileIndex);
136
- return pathPrefix + sequence + fileNameExtension;
137
- }
138
-
139
- private void putFile(Path from, String key) {
140
- PutObjectRequest request = new PutObjectRequest(bucket, key,
141
- from.toFile());
142
- client.putObject(request);
143
- }
144
-
145
- private void closeCurrent() {
146
- if (current == null) {
147
- return;
148
- }
149
-
150
- try {
151
- putFile(tempFilePath, buildCurrentKey());
152
- fileIndex++;
153
- } finally {
154
- try {
155
- current.close();
156
- current = null;
157
- } catch (IOException e) {
158
- throw new RuntimeException(e);
159
- } finally {
160
- deleteTempFile();
161
- }
162
- }
163
- }
164
-
165
- @Override
166
- public void nextFile() {
167
- closeCurrent();
168
-
169
- try {
170
- tempFilePath = newTempFile(tempPathPrefix);
171
-
172
- log.info("Writing S3 file '{}'", buildCurrentKey());
173
-
174
- current = Files.newOutputStream(tempFilePath);
175
- } catch (IOException e) {
176
- throw new RuntimeException(e);
177
- }
178
- }
179
-
180
- @Override
181
- public void add(Buffer buffer) {
182
- if (current == null) {
183
- throw new IllegalStateException(
184
- "nextFile() must be called before poll()");
185
- }
186
-
187
- try {
188
- current.write(buffer.array(), buffer.offset(), buffer.limit());
189
- } catch (IOException ex) {
190
- throw new RuntimeException(ex);
191
- } finally {
192
- buffer.release();
193
- }
194
- }
195
-
196
- @Override
197
- public void finish() {
198
- closeCurrent();
199
- }
200
-
201
- @Override
202
- public void close() {
203
- closeCurrent();
204
- }
205
-
206
- @Override
207
- public void abort() {
208
- deleteTempFile();
209
- }
210
-
211
- @Override
212
- public TaskReport commit() {
213
- TaskReport report = Exec.newTaskReport();
214
- return report;
215
- }
216
- }
217
-
218
- private void validateSequenceFormat(PluginTask task) {
219
- try {
220
- @SuppressWarnings("unused")
221
- String dontCare = String.format(Locale.ENGLISH,
222
- task.getSequenceFormat(), 0, 0);
223
- } catch (IllegalFormatException ex) {
224
- throw new ConfigException(
225
- "Invalid sequence_format: parameter for file output plugin",
226
- ex);
227
- }
228
- }
229
-
230
- @Override
231
- public ConfigDiff transaction(ConfigSource config, int taskCount,
232
- Control control) {
233
- PluginTask task = config.loadConfig(PluginTask.class);
234
-
235
- validateSequenceFormat(task);
236
-
237
- return resume(task.dump(), taskCount, control);
238
- }
239
-
240
- @Override
241
- public ConfigDiff resume(TaskSource taskSource, int taskCount,
242
- Control control) {
243
- control.run(taskSource);
244
- return Exec.newConfigDiff();
245
- }
246
-
247
- @Override
248
- public void cleanup(TaskSource taskSource, int taskCount,
249
- List<TaskReport> successTaskReports) {
250
- }
251
-
252
- @Override
253
- public TransactionalFileOutput open(TaskSource taskSource, int taskIndex) {
254
- PluginTask task = taskSource.loadTask(PluginTask.class);
255
-
256
- return new S3FileOutput(task, taskIndex);
257
- }
258
- }
1
+ package org.embulk.output;
2
+
3
+ import com.amazonaws.ClientConfiguration;
4
+ import com.amazonaws.auth.BasicAWSCredentials;
5
+ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
6
+ import com.amazonaws.services.s3.AmazonS3Client;
7
+ import com.amazonaws.services.s3.model.CannedAccessControlList;
8
+ import com.amazonaws.services.s3.model.PutObjectRequest;
9
+ import com.google.common.base.Optional;
10
+ import org.embulk.config.Config;
11
+ import org.embulk.config.ConfigDefault;
12
+ import org.embulk.config.ConfigDiff;
13
+ import org.embulk.config.ConfigException;
14
+ import org.embulk.config.ConfigSource;
15
+ import org.embulk.config.Task;
16
+ import org.embulk.config.TaskReport;
17
+ import org.embulk.config.TaskSource;
18
+ import org.embulk.spi.Buffer;
19
+ import org.embulk.spi.Exec;
20
+ import org.embulk.spi.FileOutput;
21
+ import org.embulk.spi.FileOutputPlugin;
22
+ import org.embulk.spi.TransactionalFileOutput;
23
+ import org.slf4j.Logger;
24
+
25
+ import java.io.IOException;
26
+ import java.io.OutputStream;
27
+ import java.nio.file.Files;
28
+ import java.nio.file.Path;
29
+ import java.util.IllegalFormatException;
30
+ import java.util.List;
31
+ import java.util.Locale;
32
+
33
+ public class S3FileOutputPlugin
34
+ implements FileOutputPlugin
35
+ {
36
+ public interface PluginTask
37
+ extends Task
38
+ {
39
+ @Config("path_prefix")
40
+ String getPathPrefix();
41
+
42
+ @Config("file_ext")
43
+ String getFileNameExtension();
44
+
45
+ @Config("sequence_format")
46
+ @ConfigDefault("\".%03d.%02d\"")
47
+ String getSequenceFormat();
48
+
49
+ @Config("bucket")
50
+ String getBucket();
51
+
52
+ @Config("endpoint")
53
+ @ConfigDefault("null")
54
+ Optional<String> getEndpoint();
55
+
56
+ @Config("access_key_id")
57
+ @ConfigDefault("null")
58
+ Optional<String> getAccessKeyId();
59
+
60
+ @Config("secret_access_key")
61
+ @ConfigDefault("null")
62
+ Optional<String> getSecretAccessKey();
63
+
64
+ @Config("tmp_path_prefix")
65
+ @ConfigDefault("\"embulk-output-s3-\"")
66
+ String getTempPathPrefix();
67
+
68
+ @Config("canned_acl")
69
+ @ConfigDefault("null")
70
+ Optional<CannedAccessControlList> getCannedAccessControlList();
71
+ }
72
+
73
+ public static class S3FileOutput
74
+ implements FileOutput,
75
+ TransactionalFileOutput
76
+ {
77
+ private final Logger log = Exec.getLogger(S3FileOutputPlugin.class);
78
+
79
+ private final String bucket;
80
+ private final String pathPrefix;
81
+ private final String sequenceFormat;
82
+ private final String fileNameExtension;
83
+ private final String tempPathPrefix;
84
+ private final Optional<CannedAccessControlList> cannedAccessControlListOptional;
85
+
86
+ private int taskIndex;
87
+ private int fileIndex;
88
+ private AmazonS3Client client;
89
+ private OutputStream current;
90
+ private Path tempFilePath;
91
+
92
+ private static AmazonS3Client newS3Client(PluginTask task)
93
+ {
94
+ AmazonS3Client client;
95
+
96
+ if (task.getAccessKeyId().isPresent()) {
97
+ BasicAWSCredentials basicAWSCredentials = new BasicAWSCredentials(
98
+ task.getAccessKeyId().get(), task.getSecretAccessKey().get());
99
+
100
+ ClientConfiguration config = new ClientConfiguration();
101
+ // TODO: Support more configurations.
102
+
103
+ client = new AmazonS3Client(basicAWSCredentials, config);
104
+ }
105
+ else {
106
+ if (System.getenv("AWS_ACCESS_KEY_ID") == null) {
107
+ client = new AmazonS3Client(new EnvironmentVariableCredentialsProvider());
108
+ }
109
+ else { // IAM ROLE
110
+ client = new AmazonS3Client();
111
+ }
112
+ }
113
+ if (task.getEndpoint().isPresent()) {
114
+ client.setEndpoint(task.getEndpoint().get());
115
+ }
116
+
117
+ return client;
118
+ }
119
+
120
+ public S3FileOutput(PluginTask task, int taskIndex)
121
+ {
122
+ this.taskIndex = taskIndex;
123
+ this.client = newS3Client(task);
124
+ this.bucket = task.getBucket();
125
+ this.pathPrefix = task.getPathPrefix();
126
+ this.sequenceFormat = task.getSequenceFormat();
127
+ this.fileNameExtension = task.getFileNameExtension();
128
+ this.tempPathPrefix = task.getTempPathPrefix();
129
+ this.cannedAccessControlListOptional = task.getCannedAccessControlList();
130
+ }
131
+
132
+ private static Path newTempFile(String prefix)
133
+ throws IOException
134
+ {
135
+ return Files.createTempFile(prefix, null);
136
+ }
137
+
138
+ private void deleteTempFile()
139
+ {
140
+ if (tempFilePath == null) {
141
+ return;
142
+ }
143
+
144
+ try {
145
+ Files.delete(tempFilePath);
146
+ tempFilePath = null;
147
+ }
148
+ catch (IOException e) {
149
+ throw new RuntimeException(e);
150
+ }
151
+ }
152
+
153
+ private String buildCurrentKey()
154
+ {
155
+ String sequence = String.format(sequenceFormat, taskIndex,
156
+ fileIndex);
157
+ return pathPrefix + sequence + fileNameExtension;
158
+ }
159
+
160
+ private void putFile(Path from, String key)
161
+ {
162
+ PutObjectRequest request = new PutObjectRequest(bucket, key, from.toFile());
163
+ if (cannedAccessControlListOptional.isPresent()) {
164
+ request.withCannedAcl(cannedAccessControlListOptional.get());
165
+ }
166
+ client.putObject(request);
167
+ }
168
+
169
+ private void closeCurrent()
170
+ {
171
+ if (current == null) {
172
+ return;
173
+ }
174
+
175
+ try {
176
+ putFile(tempFilePath, buildCurrentKey());
177
+ fileIndex++;
178
+ }
179
+ finally {
180
+ try {
181
+ current.close();
182
+ current = null;
183
+ }
184
+ catch (IOException e) {
185
+ throw new RuntimeException(e);
186
+ }
187
+ finally {
188
+ deleteTempFile();
189
+ }
190
+ }
191
+ }
192
+
193
+ @Override
194
+ public void nextFile()
195
+ {
196
+ closeCurrent();
197
+
198
+ try {
199
+ tempFilePath = newTempFile(tempPathPrefix);
200
+
201
+ log.info("Writing S3 file '{}'", buildCurrentKey());
202
+
203
+ current = Files.newOutputStream(tempFilePath);
204
+ }
205
+ catch (IOException e) {
206
+ throw new RuntimeException(e);
207
+ }
208
+ }
209
+
210
+ @Override
211
+ public void add(Buffer buffer)
212
+ {
213
+ if (current == null) {
214
+ throw new IllegalStateException(
215
+ "nextFile() must be called before poll()");
216
+ }
217
+
218
+ try {
219
+ current.write(buffer.array(), buffer.offset(), buffer.limit());
220
+ }
221
+ catch (IOException ex) {
222
+ throw new RuntimeException(ex);
223
+ }
224
+ finally {
225
+ buffer.release();
226
+ }
227
+ }
228
+
229
+ @Override
230
+ public void finish()
231
+ {
232
+ closeCurrent();
233
+ }
234
+
235
+ @Override
236
+ public void close()
237
+ {
238
+ closeCurrent();
239
+ }
240
+
241
+ @Override
242
+ public void abort()
243
+ {
244
+ deleteTempFile();
245
+ }
246
+
247
+ @Override
248
+ public TaskReport commit()
249
+ {
250
+ TaskReport report = Exec.newTaskReport();
251
+ return report;
252
+ }
253
+ }
254
+
255
+ private void validateSequenceFormat(PluginTask task)
256
+ {
257
+ try {
258
+ @SuppressWarnings("unused")
259
+ String dontCare = String.format(Locale.ENGLISH,
260
+ task.getSequenceFormat(), 0, 0);
261
+ }
262
+ catch (IllegalFormatException ex) {
263
+ throw new ConfigException(
264
+ "Invalid sequence_format: parameter for file output plugin",
265
+ ex);
266
+ }
267
+ }
268
+
269
+ @Override
270
+ public ConfigDiff transaction(ConfigSource config, int taskCount,
271
+ Control control)
272
+ {
273
+ PluginTask task = config.loadConfig(PluginTask.class);
274
+
275
+ validateSequenceFormat(task);
276
+
277
+ return resume(task.dump(), taskCount, control);
278
+ }
279
+
280
+ @Override
281
+ public ConfigDiff resume(TaskSource taskSource, int taskCount,
282
+ Control control)
283
+ {
284
+ control.run(taskSource);
285
+ return Exec.newConfigDiff();
286
+ }
287
+
288
+ @Override
289
+ public void cleanup(TaskSource taskSource, int taskCount,
290
+ List<TaskReport> successTaskReports)
291
+ {
292
+ }
293
+
294
+ @Override
295
+ public TransactionalFileOutput open(TaskSource taskSource, int taskIndex)
296
+ {
297
+ PluginTask task = taskSource.loadTask(PluginTask.class);
298
+
299
+ return new S3FileOutput(task, taskIndex);
300
+ }
301
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-s3
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.1.0
4
+ version: 1.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Manabu Takayama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-28 00:00:00.000000000 Z
11
+ date: 2016-03-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -49,11 +49,14 @@ files:
49
49
  - LICENSE.txt
50
50
  - README.md
51
51
  - build.gradle
52
+ - config/checkstyle/checkstyle.xml
53
+ - config/checkstyle/default.xml
52
54
  - gradle/wrapper/gradle-wrapper.jar
53
55
  - gradle/wrapper/gradle-wrapper.properties
54
56
  - gradlew
55
57
  - gradlew.bat
56
58
  - lib/embulk/output/s3.rb
59
+ - settings.gradle
57
60
  - src/main/java/org/embulk/output/S3FileOutputPlugin.java
58
61
  - src/test/java/org/embulk/output/TestS3FileOutputPlugin.java
59
62
  - classpath/aws-java-sdk-core-1.10.26.jar
@@ -61,7 +64,7 @@ files:
61
64
  - classpath/aws-java-sdk-s3-1.10.26.jar
62
65
  - classpath/commons-codec-1.6.jar
63
66
  - classpath/commons-logging-1.1.3.jar
64
- - classpath/embulk-output-s3-1.1.0.jar
67
+ - classpath/embulk-output-s3-1.2.0.jar
65
68
  - classpath/httpclient-4.3.6.jar
66
69
  - classpath/httpcore-4.3.3.jar
67
70
  homepage: https://github.com/llibra/embulk-output-s3