embulk-output-azure_blob_storage 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +4 -0
- data/README.md +0 -4
- data/build.gradle +25 -23
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +107 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java +39 -24
- data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java +20 -22
- metadata +6 -4
- data/embulk-output-azure_blob_storage.gemspec +0 -18
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8e7f5671df4655218d40ef364f9475e05f3cdaa4
|
|
4
|
+
data.tar.gz: 5ba40cd551788bdde07a5a08ea05998a3ee7f0cb
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c3330eefd6eab6363a399a09905c374c0b8c427fa367f44d545d2b7eba77c91e4480b7d76b360fa06307603532de9da227e90b5ca3380aa650087165de4c5149
|
|
7
|
+
data.tar.gz: 76c557136ad97309d1175433879ad7ae92ff2d5b826b72b772c724b624130685c2e53a748b1b079f6c3d914f90dda9ef86311a9536d2543f60cd69d546d10576
|
data/ChangeLog
ADDED
data/README.md
CHANGED
|
@@ -60,10 +60,6 @@ $ ./gradlew test # -t to watch change of files and rebuild continuously
|
|
|
60
60
|
|
|
61
61
|
To run unit tests, we need to configure the following environment variables.
|
|
62
62
|
|
|
63
|
-
Additionally, following files will be needed to upload to existing GCS bucket.
|
|
64
|
-
* [sample_01.csv](./src/test/resources/sample_01.csv)
|
|
65
|
-
* [sample_02.csv](./src/test/resources/sample_02.csv)
|
|
66
|
-
|
|
67
63
|
When environment variables are not set, skip some test cases.
|
|
68
64
|
|
|
69
65
|
```
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,7 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
5
6
|
id "jacoco"
|
|
6
7
|
}
|
|
7
8
|
import com.github.jrubygradle.JRubyExec
|
|
@@ -16,17 +17,17 @@ configurations {
|
|
|
16
17
|
sourceCompatibility = 1.7
|
|
17
18
|
targetCompatibility = 1.7
|
|
18
19
|
|
|
19
|
-
version = "0.1.
|
|
20
|
+
version = "0.1.2"
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.2"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.2"
|
|
24
25
|
|
|
25
26
|
compile "com.microsoft.azure:azure-storage:4.0.0"
|
|
26
27
|
|
|
27
28
|
testCompile "junit:junit:4.12"
|
|
28
|
-
testCompile "org.embulk:embulk-core:0.
|
|
29
|
-
testCompile "org.embulk:embulk-standards:0.
|
|
29
|
+
testCompile "org.embulk:embulk-core:0.8.2:tests"
|
|
30
|
+
testCompile "org.embulk:embulk-standards:0.8.2"
|
|
30
31
|
}
|
|
31
32
|
|
|
32
33
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
@@ -36,27 +37,30 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
36
37
|
}
|
|
37
38
|
clean { delete "classpath" }
|
|
38
39
|
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
40
|
+
checkstyle {
|
|
41
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
42
|
+
toolVersion = '6.14.1'
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
44
|
+
checkstyleMain {
|
|
45
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
46
|
+
ignoreFailures = true
|
|
47
|
+
}
|
|
48
|
+
checkstyleTest {
|
|
49
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
50
|
+
ignoreFailures = true
|
|
51
|
+
}
|
|
52
|
+
task checkstyle(type: Checkstyle) {
|
|
53
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
54
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
48
55
|
}
|
|
49
56
|
|
|
50
|
-
task
|
|
51
|
-
|
|
52
|
-
|
|
57
|
+
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
|
58
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
59
|
+
script "build/gemspec"
|
|
60
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
53
61
|
}
|
|
54
62
|
|
|
55
|
-
task gemspec {
|
|
56
|
-
ext.gemspecFile = file("${project.name}.gemspec")
|
|
57
|
-
inputs.file "build.gradle"
|
|
58
|
-
outputs.file gemspecFile
|
|
59
|
-
doLast { gemspecFile.write($/
|
|
63
|
+
task gemspec << { file("build/gemspec").write($/
|
|
60
64
|
Gem::Specification.new do |spec|
|
|
61
65
|
spec.name = "${project.name}"
|
|
62
66
|
spec.version = "${project.version}"
|
|
@@ -75,6 +79,4 @@ Gem::Specification.new do |spec|
|
|
|
75
79
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
76
80
|
end
|
|
77
81
|
/$)
|
|
78
|
-
}
|
|
79
82
|
}
|
|
80
|
-
clean { delete "${project.name}.gemspec" }
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="MethodTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
|
|
92
|
+
<module name="WhitespaceAround">
|
|
93
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
94
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
95
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
96
|
+
<property name="tokens" value="
|
|
97
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
98
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
99
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
100
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
101
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
102
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
103
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
104
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
105
|
+
</module>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
data/src/main/java/org/embulk/output/azure_blob_storage/AzureBlobStorageFileOutputPlugin.java
CHANGED
|
@@ -1,31 +1,37 @@
|
|
|
1
1
|
package org.embulk.output.azure_blob_storage;
|
|
2
2
|
|
|
3
|
-
import java.io.File;
|
|
4
|
-
import java.io.BufferedOutputStream;
|
|
5
|
-
import java.io.FileInputStream;
|
|
6
|
-
import java.io.FileOutputStream;
|
|
7
|
-
import java.io.FileNotFoundException;
|
|
8
|
-
import java.io.IOException;
|
|
9
|
-
import java.net.URISyntaxException;
|
|
10
|
-
import java.security.InvalidKeyException;
|
|
11
|
-
import java.util.List;
|
|
12
3
|
import com.google.common.base.Throwables;
|
|
4
|
+
import com.microsoft.azure.storage.CloudStorageAccount;
|
|
5
|
+
import com.microsoft.azure.storage.StorageException;
|
|
6
|
+
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
|
7
|
+
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
|
8
|
+
import com.microsoft.azure.storage.blob.CloudBlockBlob;
|
|
13
9
|
import org.embulk.config.Config;
|
|
14
|
-
import org.embulk.config.ConfigException;
|
|
15
10
|
import org.embulk.config.ConfigDefault;
|
|
16
|
-
import org.embulk.config.ConfigSource;
|
|
17
11
|
import org.embulk.config.ConfigDiff;
|
|
18
|
-
import org.embulk.config.
|
|
12
|
+
import org.embulk.config.ConfigException;
|
|
13
|
+
import org.embulk.config.ConfigSource;
|
|
19
14
|
import org.embulk.config.Task;
|
|
15
|
+
import org.embulk.config.TaskReport;
|
|
20
16
|
import org.embulk.config.TaskSource;
|
|
17
|
+
import org.embulk.spi.Buffer;
|
|
21
18
|
import org.embulk.spi.Exec;
|
|
22
19
|
import org.embulk.spi.FileOutputPlugin;
|
|
23
20
|
import org.embulk.spi.TransactionalFileOutput;
|
|
24
|
-
|
|
25
|
-
import com.microsoft.azure.storage.*;
|
|
26
|
-
import com.microsoft.azure.storage.blob.*;
|
|
21
|
+
|
|
27
22
|
import org.slf4j.Logger;
|
|
28
23
|
|
|
24
|
+
import java.io.BufferedOutputStream;
|
|
25
|
+
import java.io.File;
|
|
26
|
+
import java.io.FileInputStream;
|
|
27
|
+
import java.io.FileNotFoundException;
|
|
28
|
+
import java.io.FileOutputStream;
|
|
29
|
+
|
|
30
|
+
import java.io.IOException;
|
|
31
|
+
import java.net.URISyntaxException;
|
|
32
|
+
import java.security.InvalidKeyException;
|
|
33
|
+
import java.util.List;
|
|
34
|
+
|
|
29
35
|
public class AzureBlobStorageFileOutputPlugin
|
|
30
36
|
implements FileOutputPlugin
|
|
31
37
|
{
|
|
@@ -68,7 +74,8 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
68
74
|
log.info(String.format("container [%s] is not exists and created.", containerName));
|
|
69
75
|
container.createIfNotExists();
|
|
70
76
|
}
|
|
71
|
-
}
|
|
77
|
+
}
|
|
78
|
+
catch (StorageException | URISyntaxException | ConfigException ex) {
|
|
72
79
|
Throwables.propagate(ex);
|
|
73
80
|
}
|
|
74
81
|
|
|
@@ -97,14 +104,16 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
97
104
|
CloudStorageAccount account;
|
|
98
105
|
try {
|
|
99
106
|
account = CloudStorageAccount.parse(connectionString);
|
|
100
|
-
}
|
|
107
|
+
}
|
|
108
|
+
catch (InvalidKeyException | URISyntaxException ex) {
|
|
101
109
|
throw new ConfigException(ex.getMessage());
|
|
102
110
|
}
|
|
103
111
|
return account.createCloudBlobClient();
|
|
104
112
|
}
|
|
105
113
|
|
|
106
114
|
@Override
|
|
107
|
-
public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
|
|
115
|
+
public TransactionalFileOutput open(TaskSource taskSource, final int taskIndex)
|
|
116
|
+
{
|
|
108
117
|
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
|
109
118
|
return new AzureFileOutput(task, taskIndex);
|
|
110
119
|
}
|
|
@@ -131,7 +140,8 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
131
140
|
this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
|
|
132
141
|
try {
|
|
133
142
|
this.container = client.getContainerReference(task.getContainer());
|
|
134
|
-
}
|
|
143
|
+
}
|
|
144
|
+
catch (URISyntaxException | StorageException ex) {
|
|
135
145
|
Throwables.propagate(ex);
|
|
136
146
|
}
|
|
137
147
|
}
|
|
@@ -156,7 +166,8 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
156
166
|
}
|
|
157
167
|
log.info(String.format("Writing local file [%s]", filePath));
|
|
158
168
|
output = new BufferedOutputStream(new FileOutputStream(filePath));
|
|
159
|
-
}
|
|
169
|
+
}
|
|
170
|
+
catch (FileNotFoundException ex) {
|
|
160
171
|
throw Throwables.propagate(ex);
|
|
161
172
|
}
|
|
162
173
|
}
|
|
@@ -167,7 +178,8 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
167
178
|
try {
|
|
168
179
|
output.close();
|
|
169
180
|
fileIndex++;
|
|
170
|
-
}
|
|
181
|
+
}
|
|
182
|
+
catch (IOException ex) {
|
|
171
183
|
throw Throwables.propagate(ex);
|
|
172
184
|
}
|
|
173
185
|
}
|
|
@@ -178,9 +190,11 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
178
190
|
{
|
|
179
191
|
try {
|
|
180
192
|
output.write(buffer.array(), buffer.offset(), buffer.limit());
|
|
181
|
-
}
|
|
193
|
+
}
|
|
194
|
+
catch (IOException ex) {
|
|
182
195
|
throw Throwables.propagate(ex);
|
|
183
|
-
}
|
|
196
|
+
}
|
|
197
|
+
finally {
|
|
184
198
|
buffer.release();
|
|
185
199
|
}
|
|
186
200
|
}
|
|
@@ -197,7 +211,8 @@ public class AzureBlobStorageFileOutputPlugin
|
|
|
197
211
|
log.info(String.format("Upload completed [%s]", filePath));
|
|
198
212
|
file.delete();
|
|
199
213
|
log.info(String.format("Delete completed local file [%s]", filePath));
|
|
200
|
-
}
|
|
214
|
+
}
|
|
215
|
+
catch (StorageException | URISyntaxException | IOException ex) {
|
|
201
216
|
Throwables.propagate(ex);
|
|
202
217
|
}
|
|
203
218
|
}
|
data/src/test/java/org/embulk/output/azure_blob_storage/TestAzureBlobStorageFileOutputPlugin.java
CHANGED
|
@@ -1,45 +1,43 @@
|
|
|
1
1
|
package org.embulk.output.azure_blob_storage;
|
|
2
2
|
|
|
3
|
-
import java.io.BufferedReader;
|
|
4
|
-
import java.io.ByteArrayOutputStream;
|
|
5
|
-
import java.io.FileInputStream;
|
|
6
|
-
import java.io.IOException;
|
|
7
|
-
import java.io.InputStream;
|
|
8
|
-
import java.io.InputStreamReader;
|
|
9
|
-
import java.lang.reflect.Method;
|
|
10
|
-
import java.security.GeneralSecurityException;
|
|
11
|
-
import java.util.Arrays;
|
|
12
|
-
import java.util.List;
|
|
13
|
-
|
|
14
3
|
import com.google.common.collect.ImmutableList;
|
|
15
4
|
import com.google.common.collect.ImmutableMap;
|
|
16
5
|
import com.google.common.collect.Lists;
|
|
17
|
-
|
|
6
|
+
import com.microsoft.azure.storage.blob.CloudBlob;
|
|
7
|
+
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
|
8
|
+
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
|
18
9
|
import org.embulk.EmbulkTestRuntime;
|
|
19
10
|
import org.embulk.config.ConfigDiff;
|
|
20
11
|
import org.embulk.config.ConfigSource;
|
|
21
12
|
import org.embulk.config.TaskReport;
|
|
22
13
|
import org.embulk.config.TaskSource;
|
|
14
|
+
import org.embulk.output.azure_blob_storage.AzureBlobStorageFileOutputPlugin.PluginTask;
|
|
23
15
|
import org.embulk.spi.Buffer;
|
|
24
16
|
import org.embulk.spi.Exec;
|
|
25
|
-
import org.embulk.spi.Schema;
|
|
26
|
-
import org.embulk.spi.OutputPlugin;
|
|
27
|
-
import org.embulk.spi.FileOutputRunner;
|
|
28
17
|
import org.embulk.spi.FileOutputPlugin;
|
|
18
|
+
import org.embulk.spi.FileOutputRunner;
|
|
19
|
+
import org.embulk.spi.OutputPlugin;
|
|
20
|
+
import org.embulk.spi.Schema;
|
|
29
21
|
import org.embulk.spi.TransactionalFileOutput;
|
|
30
22
|
import org.embulk.standards.CsvParserPlugin;
|
|
31
23
|
|
|
32
|
-
import org.junit.BeforeClass;
|
|
33
24
|
import org.junit.Before;
|
|
25
|
+
import org.junit.BeforeClass;
|
|
34
26
|
import org.junit.Rule;
|
|
35
27
|
import org.junit.Test;
|
|
36
28
|
import static org.junit.Assert.assertEquals;
|
|
37
29
|
import static org.junit.Assume.assumeNotNull;
|
|
38
30
|
|
|
39
|
-
import
|
|
40
|
-
import
|
|
41
|
-
import
|
|
42
|
-
import
|
|
31
|
+
import java.io.BufferedReader;
|
|
32
|
+
import java.io.ByteArrayOutputStream;
|
|
33
|
+
import java.io.FileInputStream;
|
|
34
|
+
import java.io.IOException;
|
|
35
|
+
import java.io.InputStream;
|
|
36
|
+
import java.io.InputStreamReader;
|
|
37
|
+
import java.lang.reflect.Method;
|
|
38
|
+
import java.security.GeneralSecurityException;
|
|
39
|
+
import java.util.Arrays;
|
|
40
|
+
import java.util.List;
|
|
43
41
|
|
|
44
42
|
public class TestAzureBlobStorageFileOutputPlugin
|
|
45
43
|
{
|
|
@@ -367,9 +365,9 @@ public class TestAzureBlobStorageFileOutputPlugin
|
|
|
367
365
|
{
|
|
368
366
|
ByteArrayOutputStream bo = new ByteArrayOutputStream();
|
|
369
367
|
byte [] buffer = new byte[1024];
|
|
370
|
-
while(true) {
|
|
368
|
+
while (true) {
|
|
371
369
|
int len = is.read(buffer);
|
|
372
|
-
if(len < 0) {
|
|
370
|
+
if (len < 0) {
|
|
373
371
|
break;
|
|
374
372
|
}
|
|
375
373
|
bo.write(buffer, 0, len);
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-azure_blob_storage
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Satoshi Akama
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-02-02 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -47,9 +47,11 @@ extra_rdoc_files: []
|
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
49
|
- .travis.yml
|
|
50
|
+
- ChangeLog
|
|
50
51
|
- README.md
|
|
51
52
|
- build.gradle
|
|
52
|
-
-
|
|
53
|
+
- config/checkstyle/checkstyle.xml
|
|
54
|
+
- config/checkstyle/default.xml
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.jar
|
|
54
56
|
- gradle/wrapper/gradle-wrapper.properties
|
|
55
57
|
- gradlew
|
|
@@ -61,7 +63,7 @@ files:
|
|
|
61
63
|
- src/test/resources/sample_02.csv
|
|
62
64
|
- classpath/azure-storage-4.0.0.jar
|
|
63
65
|
- classpath/commons-lang3-3.4.jar
|
|
64
|
-
- classpath/embulk-output-azure_blob_storage-0.1.
|
|
66
|
+
- classpath/embulk-output-azure_blob_storage-0.1.2.jar
|
|
65
67
|
- classpath/jackson-core-2.6.0.jar
|
|
66
68
|
homepage: https://github.com/sakama/embulk-output-azure_blob_storage
|
|
67
69
|
licenses:
|
|
@@ -1,18 +0,0 @@
|
|
|
1
|
-
|
|
2
|
-
Gem::Specification.new do |spec|
|
|
3
|
-
spec.name = "embulk-output-azure_blob_storage"
|
|
4
|
-
spec.version = "0.1.1"
|
|
5
|
-
spec.authors = ["Satoshi Akama"]
|
|
6
|
-
spec.summary = %[Microsoft Azure blob Storage file output plugin for Embulk]
|
|
7
|
-
spec.description = %[Stores files on Microsoft Azure blob Storage.]
|
|
8
|
-
spec.email = ["satoshiakama@gmail.com"]
|
|
9
|
-
spec.licenses = ["Apache-2.0"]
|
|
10
|
-
spec.homepage = "https://github.com/sakama/embulk-output-azure_blob_storage"
|
|
11
|
-
|
|
12
|
-
spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
|
|
13
|
-
spec.test_files = spec.files.grep(%r"^(test|spec)/")
|
|
14
|
-
spec.require_paths = ["lib"]
|
|
15
|
-
|
|
16
|
-
spec.add_development_dependency 'bundler', ['~> 1.0']
|
|
17
|
-
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
18
|
-
end
|