embulk-input-azure_blob_storage 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +13 -0
- data/README.md +53 -0
- data/build.gradle +27 -22
- data/classpath/embulk-input-azure_blob_storage-0.1.3.jar +0 -0
- data/config/checkstyle/checkstyle.xml +130 -0
- data/config/checkstyle/default.xml +110 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java +63 -30
- data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java +338 -0
- data/src/test/resources/sample_01.csv +6 -0
- data/src/test/resources/sample_02.csv +6 -0
- metadata +8 -4
- data/ChangeLog +0 -4
- data/classpath/embulk-input-azure_blob_storage-0.1.2.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a85c5e311d426a1f67c47b4947683827a3bb2140
|
|
4
|
+
data.tar.gz: 2ae6c9a16e143e4731e4de7518fa504d2cb2f8c1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 68588df49d434087541ea542a4731236d5496356df5b66bdb81f620803c305f76c33b39ff7340d176378f80c0f41e0275efedc95b180198828776218a3148f55
|
|
7
|
+
data.tar.gz: 7cc8c4758c67360d7cbbd5cf4ca12bfe8f8b975641fcc31fa2b3308e4ba3c8093e7aa0edf577e078d9f3441e8636bae700dc07fbb16c0b181ec71742b3ae963c
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
## 0.1.3 - 2015-03-16
|
|
2
|
+
|
|
3
|
+
* [maintenance] Add unit test[#4](https://github.com/sakama/embulk-input-azure_blob_storage/pull/4)
|
|
4
|
+
* [maintenance] Add retry logic [#3](https://github.com/sakama/embulk-input-azure_blob_storage/pull/3)
|
|
5
|
+
|
|
6
|
+
## 0.1.2 - 2015-10-11
|
|
7
|
+
|
|
8
|
+
* [maintenance] Upgrade embulk to v0.8.2 [#2](https://github.com/sakama/embulk-input-azure_blob_storage/pull/2)
|
|
9
|
+
|
|
10
|
+
## 0.1.1 - 2015-10-11
|
|
11
|
+
|
|
12
|
+
* [maintenance] Change 'DefaultEndpointsProtocol' from http to https [#1](https://github.com/sakama/embulk-input-azure_blob_storage/pull/1)
|
|
13
|
+
* [maintenance] Specify targetCompatibility = 1.7
|
data/README.md
CHANGED
|
@@ -60,3 +60,56 @@ out: {type: stdout}
|
|
|
60
60
|
```
|
|
61
61
|
$ ./gradlew gem # -t to watch change of files and rebuild continuously
|
|
62
62
|
```
|
|
63
|
+
|
|
64
|
+
## Test
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
$ ./gradlew test # -t to watch change of files and rebuild continuously
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
To run unit tests, we need to configure the following environment variables.
|
|
71
|
+
|
|
72
|
+
Additionally, following files will be needed to upload to existing GCS bucket.
|
|
73
|
+
|
|
74
|
+
* [sample_01.csv](src/test/resources/sample_01.csv)
|
|
75
|
+
* [sample_02.csv](src/test/resources/sample_02.csv)
|
|
76
|
+
|
|
77
|
+
When environment variables are not set, skip some test cases.
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
AZURE_ACCOUNT_NAME
|
|
81
|
+
AZURE_ACCOUNT_KEY
|
|
82
|
+
AZURE_CONTAINER
|
|
83
|
+
AZURE_CONTAINER_IMPORT_DIRECTORY (optional, if needed)
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
|
|
87
|
+
```xml
|
|
88
|
+
$ vi ~/Library/LaunchAgents/environment.plist
|
|
89
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
90
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
91
|
+
<plist version="1.0">
|
|
92
|
+
<dict>
|
|
93
|
+
<key>Label</key>
|
|
94
|
+
<string>my.startup</string>
|
|
95
|
+
<key>ProgramArguments</key>
|
|
96
|
+
<array>
|
|
97
|
+
<string>sh</string>
|
|
98
|
+
<string>-c</string>
|
|
99
|
+
<string>
|
|
100
|
+
launchctl setenv AZURE_ACCOUNT_NAME my-account-name
|
|
101
|
+
launchctl setenv AZURE_ACCOUNT_KEY my-account-key
|
|
102
|
+
launchctl setenv AZURE_CONTAINER my-container
|
|
103
|
+
launchctl setenv AZURE_CONTAINER_IMPORT_DIRECTORY unittests
|
|
104
|
+
</string>
|
|
105
|
+
</array>
|
|
106
|
+
<key>RunAtLoad</key>
|
|
107
|
+
<true/>
|
|
108
|
+
</dict>
|
|
109
|
+
</plist>
|
|
110
|
+
|
|
111
|
+
$ launchctl load ~/Library/LaunchAgents/environment.plist
|
|
112
|
+
$ launchctl getenv AZURE_ACCOUNT_NAME //try to get value.
|
|
113
|
+
|
|
114
|
+
Then start your applications.
|
|
115
|
+
```
|
data/build.gradle
CHANGED
|
@@ -2,6 +2,8 @@ plugins {
|
|
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
|
4
4
|
id "java"
|
|
5
|
+
id "checkstyle"
|
|
6
|
+
id "jacoco"
|
|
5
7
|
}
|
|
6
8
|
import com.github.jrubygradle.JRubyExec
|
|
7
9
|
repositories {
|
|
@@ -15,15 +17,17 @@ configurations {
|
|
|
15
17
|
sourceCompatibility = 1.7
|
|
16
18
|
targetCompatibility = 1.7
|
|
17
19
|
|
|
18
|
-
version = "0.1.
|
|
20
|
+
version = "0.1.3"
|
|
19
21
|
|
|
20
22
|
dependencies {
|
|
21
|
-
compile "org.embulk:embulk-core:0.
|
|
22
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.2"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.2"
|
|
23
25
|
|
|
24
26
|
compile "com.microsoft.azure:azure-storage:4.0.0"
|
|
25
27
|
|
|
26
|
-
testCompile "junit:junit:4
|
|
28
|
+
testCompile "junit:junit:4.12"
|
|
29
|
+
testCompile "org.embulk:embulk-core:0.8.8:tests"
|
|
30
|
+
testCompile "org.embulk:embulk-standards:0.8.8"
|
|
27
31
|
}
|
|
28
32
|
|
|
29
33
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
@@ -33,27 +37,30 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
33
37
|
}
|
|
34
38
|
clean { delete "classpath" }
|
|
35
39
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
40
|
+
checkstyle {
|
|
41
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
42
|
+
toolVersion = '6.14.1'
|
|
40
43
|
}
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
44
|
+
checkstyleMain {
|
|
45
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
46
|
+
ignoreFailures = true
|
|
47
|
+
}
|
|
48
|
+
checkstyleTest {
|
|
49
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
50
|
+
ignoreFailures = true
|
|
51
|
+
}
|
|
52
|
+
task checkstyle(type: Checkstyle) {
|
|
53
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
54
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
45
55
|
}
|
|
46
56
|
|
|
47
|
-
task
|
|
48
|
-
|
|
49
|
-
|
|
57
|
+
task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
|
|
58
|
+
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
59
|
+
script "build/gemspec"
|
|
60
|
+
doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
|
|
50
61
|
}
|
|
51
62
|
|
|
52
|
-
task gemspec {
|
|
53
|
-
ext.gemspecFile = file("${project.name}.gemspec")
|
|
54
|
-
inputs.file "build.gradle"
|
|
55
|
-
outputs.file gemspecFile
|
|
56
|
-
doLast { gemspecFile.write($/
|
|
63
|
+
task gemspec << { file("build/gemspec").write($/
|
|
57
64
|
Gem::Specification.new do |spec|
|
|
58
65
|
spec.name = "${project.name}"
|
|
59
66
|
spec.version = "${project.version}"
|
|
@@ -72,6 +79,4 @@ Gem::Specification.new do |spec|
|
|
|
72
79
|
spec.add_development_dependency 'rake', ['>= 10.0']
|
|
73
80
|
end
|
|
74
81
|
/$)
|
|
75
|
-
}
|
|
76
82
|
}
|
|
77
|
-
clean { delete "${project.name}.gemspec" }
|
|
Binary file
|
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName">
|
|
96
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
97
|
+
</module>
|
|
98
|
+
<module name="ClassTypeParameterName">
|
|
99
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
100
|
+
</module>
|
|
101
|
+
<module name="MethodTypeParameterName">
|
|
102
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
103
|
+
</module>
|
|
104
|
+
|
|
105
|
+
<module name="AvoidStarImport"/>
|
|
106
|
+
<module name="RedundantImport"/>
|
|
107
|
+
<module name="UnusedImports"/>
|
|
108
|
+
<module name="ImportOrder">
|
|
109
|
+
<property name="groups" value="*,javax,java"/>
|
|
110
|
+
<property name="separated" value="true"/>
|
|
111
|
+
<property name="option" value="bottom"/>
|
|
112
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
113
|
+
</module>
|
|
114
|
+
|
|
115
|
+
<module name="WhitespaceAround">
|
|
116
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
117
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
118
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
119
|
+
<property name="tokens" value="
|
|
120
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
121
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
122
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
123
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
124
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
125
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
126
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
127
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
128
|
+
</module>
|
|
129
|
+
</module>
|
|
130
|
+
</module>
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName">
|
|
86
|
+
<property name="format" value="^[A-Z][A-Z0-9]*(_[A-Z0-9]+)*$"/>
|
|
87
|
+
</module>
|
|
88
|
+
<module name="ClassTypeParameterName">
|
|
89
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
90
|
+
</module>
|
|
91
|
+
<module name="MethodTypeParameterName">
|
|
92
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
93
|
+
</module>
|
|
94
|
+
|
|
95
|
+
<module name="WhitespaceAround">
|
|
96
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
97
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
98
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
99
|
+
<property name="tokens" value="
|
|
100
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
101
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
102
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
103
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
104
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
105
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
106
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
107
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
108
|
+
</module>
|
|
109
|
+
</module>
|
|
110
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -1,33 +1,40 @@
|
|
|
1
1
|
package org.embulk.input.azure_blob_storage;
|
|
2
2
|
|
|
3
|
-
import java.util.List;
|
|
4
|
-
import java.util.ArrayList;
|
|
5
|
-
import java.util.Collections;
|
|
6
|
-
import java.io.InputStream;
|
|
7
|
-
import java.io.IOException;
|
|
8
|
-
import java.security.InvalidKeyException;
|
|
9
|
-
import java.net.URISyntaxException;
|
|
10
3
|
import com.google.common.base.Optional;
|
|
11
|
-
import com.google.common.collect.ImmutableList;
|
|
12
4
|
import com.google.common.base.Throwables;
|
|
5
|
+
import com.google.common.collect.ImmutableList;
|
|
6
|
+
import com.microsoft.azure.storage.CloudStorageAccount;
|
|
7
|
+
import com.microsoft.azure.storage.ResultContinuation;
|
|
8
|
+
import com.microsoft.azure.storage.ResultSegment;
|
|
9
|
+
import com.microsoft.azure.storage.StorageException;
|
|
10
|
+
import com.microsoft.azure.storage.blob.CloudBlob;
|
|
11
|
+
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
|
12
|
+
import com.microsoft.azure.storage.blob.CloudBlobContainer;
|
|
13
|
+
import com.microsoft.azure.storage.blob.ListBlobItem;
|
|
13
14
|
import org.embulk.config.Config;
|
|
15
|
+
import org.embulk.config.ConfigDefault;
|
|
16
|
+
import org.embulk.config.ConfigDiff;
|
|
14
17
|
import org.embulk.config.ConfigException;
|
|
15
18
|
import org.embulk.config.ConfigInject;
|
|
16
|
-
import org.embulk.config.ConfigDefault;
|
|
17
|
-
import org.embulk.config.Task;
|
|
18
|
-
import org.embulk.config.TaskSource;
|
|
19
19
|
import org.embulk.config.ConfigSource;
|
|
20
|
-
import org.embulk.config.
|
|
20
|
+
import org.embulk.config.Task;
|
|
21
21
|
import org.embulk.config.TaskReport;
|
|
22
|
+
import org.embulk.config.TaskSource;
|
|
23
|
+
import org.embulk.spi.BufferAllocator;
|
|
22
24
|
import org.embulk.spi.Exec;
|
|
23
25
|
import org.embulk.spi.FileInputPlugin;
|
|
24
|
-
import org.embulk.spi.BufferAllocator;
|
|
25
26
|
import org.embulk.spi.TransactionalFileInput;
|
|
26
27
|
import org.embulk.spi.util.InputStreamFileInput;
|
|
27
|
-
import com.microsoft.azure.storage.*;
|
|
28
|
-
import com.microsoft.azure.storage.blob.*;
|
|
29
28
|
import org.slf4j.Logger;
|
|
30
29
|
|
|
30
|
+
import java.io.IOException;
|
|
31
|
+
import java.io.InputStream;
|
|
32
|
+
import java.net.URISyntaxException;
|
|
33
|
+
import java.security.InvalidKeyException;
|
|
34
|
+
import java.util.ArrayList;
|
|
35
|
+
import java.util.Collections;
|
|
36
|
+
import java.util.List;
|
|
37
|
+
|
|
31
38
|
public class AzureBlobStorageFileInputPlugin
|
|
32
39
|
implements FileInputPlugin
|
|
33
40
|
{
|
|
@@ -54,6 +61,10 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
54
61
|
@ConfigDefault("5000")
|
|
55
62
|
int getMaxResults();
|
|
56
63
|
|
|
64
|
+
@Config("max_connection_retry")
|
|
65
|
+
@ConfigDefault("5") // 5 times retry to connect sftp server if failed.
|
|
66
|
+
int getMaxConnectionRetry();
|
|
67
|
+
|
|
57
68
|
List<String> getFiles();
|
|
58
69
|
|
|
59
70
|
void setFiles(List<String> files);
|
|
@@ -84,12 +95,13 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
84
95
|
|
|
85
96
|
ConfigDiff configDiff = Exec.newConfigDiff();
|
|
86
97
|
|
|
87
|
-
List<String> files = new ArrayList<>
|
|
98
|
+
List<String> files = new ArrayList<>(task.getFiles());
|
|
88
99
|
if (files.isEmpty()) {
|
|
89
100
|
if (task.getLastPath().isPresent()) {
|
|
90
101
|
configDiff.set("last_path", task.getLastPath().get());
|
|
91
102
|
}
|
|
92
|
-
}
|
|
103
|
+
}
|
|
104
|
+
else {
|
|
93
105
|
Collections.sort(files);
|
|
94
106
|
configDiff.set("last_path", files.get(files.size() - 1));
|
|
95
107
|
}
|
|
@@ -110,7 +122,8 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
110
122
|
CloudStorageAccount account;
|
|
111
123
|
try {
|
|
112
124
|
account = CloudStorageAccount.parse(connectionString);
|
|
113
|
-
}
|
|
125
|
+
}
|
|
126
|
+
catch (InvalidKeyException | URISyntaxException ex) {
|
|
114
127
|
throw new ConfigException(ex);
|
|
115
128
|
}
|
|
116
129
|
return account.createCloudBlobClient();
|
|
@@ -138,7 +151,7 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
138
151
|
ResultSegment<ListBlobItem> blobs;
|
|
139
152
|
do {
|
|
140
153
|
blobs = container.listBlobsSegmented(prefix, true, null, maxResults, token, null, null);
|
|
141
|
-
log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength()
|
|
154
|
+
log.debug(String.format("result count(include directory):%s continuationToken:%s", blobs.getLength(), blobs.getContinuationToken()));
|
|
142
155
|
for (ListBlobItem blobItem : blobs.getResults()) {
|
|
143
156
|
if (blobItem instanceof CloudBlob) {
|
|
144
157
|
CloudBlob blob = (CloudBlob) blobItem;
|
|
@@ -150,7 +163,8 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
150
163
|
}
|
|
151
164
|
token = blobs.getContinuationToken();
|
|
152
165
|
} while (blobs.getContinuationToken() != null);
|
|
153
|
-
}
|
|
166
|
+
}
|
|
167
|
+
catch (URISyntaxException | StorageException ex) {
|
|
154
168
|
throw Throwables.propagate(ex);
|
|
155
169
|
}
|
|
156
170
|
return builder.build();
|
|
@@ -167,13 +181,14 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
167
181
|
extends InputStreamFileInput
|
|
168
182
|
implements TransactionalFileInput
|
|
169
183
|
{
|
|
170
|
-
public AzureFileInput
|
|
184
|
+
public AzureFileInput(PluginTask task, int taskIndex)
|
|
171
185
|
{
|
|
172
186
|
super(task.getBufferAllocator(), new SingleFileProvider(task, taskIndex));
|
|
173
187
|
}
|
|
174
188
|
public void abort() {}
|
|
175
189
|
|
|
176
|
-
public TaskReport commit()
|
|
190
|
+
public TaskReport commit()
|
|
191
|
+
{
|
|
177
192
|
return Exec.newTaskReport();
|
|
178
193
|
}
|
|
179
194
|
|
|
@@ -187,6 +202,7 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
187
202
|
private CloudBlobClient client;
|
|
188
203
|
private final String containerName;
|
|
189
204
|
private final String key;
|
|
205
|
+
private final int maxConnectionRetry;
|
|
190
206
|
private boolean opened = false;
|
|
191
207
|
|
|
192
208
|
public SingleFileProvider(PluginTask task, int taskIndex)
|
|
@@ -194,6 +210,7 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
194
210
|
this.client = newAzureClient(task.getAccountName(), task.getAccountKey());
|
|
195
211
|
this.containerName = task.getContainer();
|
|
196
212
|
this.key = task.getFiles().get(taskIndex);
|
|
213
|
+
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
|
197
214
|
}
|
|
198
215
|
|
|
199
216
|
@Override
|
|
@@ -203,15 +220,31 @@ public class AzureBlobStorageFileInputPlugin
|
|
|
203
220
|
return null;
|
|
204
221
|
}
|
|
205
222
|
opened = true;
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
223
|
+
int count = 0;
|
|
224
|
+
|
|
225
|
+
while (true) {
|
|
226
|
+
try {
|
|
227
|
+
CloudBlobContainer container = client.getContainerReference(containerName);
|
|
228
|
+
CloudBlob blob = container.getBlockBlobReference(key);
|
|
229
|
+
return blob.openInputStream();
|
|
230
|
+
}
|
|
231
|
+
catch (StorageException | URISyntaxException ex) {
|
|
232
|
+
if (++count == maxConnectionRetry) {
|
|
233
|
+
Throwables.propagate(ex);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
try {
|
|
237
|
+
long sleepTime = ((long) Math.pow(2, count) * 1000);
|
|
238
|
+
log.warn("Sleep in next connection retry: {} milliseconds", sleepTime);
|
|
239
|
+
Thread.sleep(sleepTime);
|
|
240
|
+
}
|
|
241
|
+
catch (InterruptedException ex2) {
|
|
242
|
+
// Ignore this exception because this exception is just about `sleep`.
|
|
243
|
+
log.warn(ex2.getMessage(), ex2);
|
|
244
|
+
}
|
|
245
|
+
log.warn("Retrying to connect Azure server: " + count + " times");
|
|
246
|
+
}
|
|
213
247
|
}
|
|
214
|
-
return inputStream;
|
|
215
248
|
}
|
|
216
249
|
|
|
217
250
|
@Override
|
data/src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java
CHANGED
|
@@ -1,5 +1,343 @@
|
|
|
1
1
|
package org.embulk.input.azure_blob_storage;
|
|
2
2
|
|
|
3
|
+
import com.google.common.collect.ImmutableList;
|
|
4
|
+
import com.google.common.collect.ImmutableMap;
|
|
5
|
+
import com.google.common.collect.Lists;
|
|
6
|
+
import com.microsoft.azure.storage.blob.CloudBlobClient;
|
|
7
|
+
import org.embulk.EmbulkTestRuntime;
|
|
8
|
+
import org.embulk.config.ConfigDiff;
|
|
9
|
+
import org.embulk.config.ConfigException;
|
|
10
|
+
import org.embulk.config.ConfigSource;
|
|
11
|
+
import org.embulk.config.TaskReport;
|
|
12
|
+
import org.embulk.config.TaskSource;
|
|
13
|
+
import org.embulk.input.azure_blob_storage.AzureBlobStorageFileInputPlugin.PluginTask;
|
|
14
|
+
import org.embulk.spi.Exec;
|
|
15
|
+
import org.embulk.spi.FileInputPlugin;
|
|
16
|
+
import org.embulk.spi.FileInputRunner;
|
|
17
|
+
import org.embulk.spi.InputPlugin;
|
|
18
|
+
import org.embulk.spi.Schema;
|
|
19
|
+
import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
20
|
+
import org.embulk.spi.util.Pages;
|
|
21
|
+
import org.embulk.standards.CsvParserPlugin;
|
|
22
|
+
import org.junit.Before;
|
|
23
|
+
import org.junit.BeforeClass;
|
|
24
|
+
import org.junit.Rule;
|
|
25
|
+
import org.junit.Test;
|
|
26
|
+
|
|
27
|
+
import java.io.IOException;
|
|
28
|
+
import java.lang.reflect.InvocationTargetException;
|
|
29
|
+
import java.lang.reflect.Method;
|
|
30
|
+
import java.security.GeneralSecurityException;
|
|
31
|
+
import java.util.ArrayList;
|
|
32
|
+
import java.util.Arrays;
|
|
33
|
+
import java.util.List;
|
|
34
|
+
|
|
35
|
+
import static org.junit.Assert.assertEquals;
|
|
36
|
+
import static org.junit.Assume.assumeNotNull;
|
|
37
|
+
|
|
3
38
|
public class TestAzureBlobStorageFileInputPlugin
|
|
4
39
|
{
|
|
40
|
+
private static String AZURE_ACCOUNT_NAME;
|
|
41
|
+
private static String AZURE_ACCOUNT_KEY;
|
|
42
|
+
private static String AZURE_CONTAINER;
|
|
43
|
+
private static String AZURE_CONTAINER_IMPORT_DIRECTORY;
|
|
44
|
+
private static String AZURE_PATH_PREFIX;
|
|
45
|
+
private FileInputRunner runner;
|
|
46
|
+
private MockPageOutput output;
|
|
47
|
+
|
|
48
|
+
/*
|
|
49
|
+
* This test case requires environment variables
|
|
50
|
+
* AZURE_ACCOUNT_NAME
|
|
51
|
+
* AZURE_ACCOUNT_KEY
|
|
52
|
+
* AZURE_CONTAINER
|
|
53
|
+
* AZURE_CONTAINER_IMPORT_DIRECTORY
|
|
54
|
+
*/
|
|
55
|
+
@BeforeClass
|
|
56
|
+
public static void initializeConstant()
|
|
57
|
+
{
|
|
58
|
+
AZURE_ACCOUNT_NAME = System.getenv("AZURE_ACCOUNT_NAME");
|
|
59
|
+
AZURE_ACCOUNT_KEY = System.getenv("AZURE_ACCOUNT_KEY");
|
|
60
|
+
AZURE_CONTAINER = System.getenv("AZURE_CONTAINER");
|
|
61
|
+
// skip test cases, if environment variables are not set.
|
|
62
|
+
assumeNotNull(AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY, AZURE_CONTAINER);
|
|
63
|
+
|
|
64
|
+
AZURE_CONTAINER_IMPORT_DIRECTORY = System.getenv("AZURE_CONTAINER_IMPORT_DIRECTORY") != null ? getDirectory(System.getenv("AZURE_CONTAINER_IMPORT_DIRECTORY")) : getDirectory("");
|
|
65
|
+
AZURE_PATH_PREFIX = AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_";
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
@Rule
|
|
69
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
|
70
|
+
private ConfigSource config;
|
|
71
|
+
private AzureBlobStorageFileInputPlugin plugin;
|
|
72
|
+
|
|
73
|
+
@Before
|
|
74
|
+
public void createResources() throws GeneralSecurityException, NoSuchMethodException, IOException
|
|
75
|
+
{
|
|
76
|
+
config = config();
|
|
77
|
+
plugin = new AzureBlobStorageFileInputPlugin();
|
|
78
|
+
runner = new FileInputRunner(runtime.getInstance(AzureBlobStorageFileInputPlugin.class));
|
|
79
|
+
output = new MockPageOutput();
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
@Test
|
|
83
|
+
public void checkDefaultValues()
|
|
84
|
+
{
|
|
85
|
+
ConfigSource config = Exec.newConfigSource()
|
|
86
|
+
.set("account_name", AZURE_ACCOUNT_NAME)
|
|
87
|
+
.set("account_key", AZURE_ACCOUNT_KEY)
|
|
88
|
+
.set("container", AZURE_CONTAINER)
|
|
89
|
+
.set("path_prefix", "my-prefix");
|
|
90
|
+
|
|
91
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
92
|
+
assertEquals(5000, task.getMaxResults());
|
|
93
|
+
assertEquals(5, task.getMaxConnectionRetry());
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
public ConfigSource config()
|
|
97
|
+
{
|
|
98
|
+
return Exec.newConfigSource()
|
|
99
|
+
.set("account_name", AZURE_ACCOUNT_NAME)
|
|
100
|
+
.set("account_key", AZURE_ACCOUNT_KEY)
|
|
101
|
+
.set("container", AZURE_CONTAINER)
|
|
102
|
+
.set("path_prefix", AZURE_PATH_PREFIX)
|
|
103
|
+
.set("last_path", "")
|
|
104
|
+
.set("file_ext", ".csv")
|
|
105
|
+
.set("parser", parserConfig(schemaConfig()));
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
@Test(expected = ConfigException.class)
|
|
109
|
+
public void checkDefaultValuesAccountNameIsNull()
|
|
110
|
+
{
|
|
111
|
+
ConfigSource config = Exec.newConfigSource()
|
|
112
|
+
.set("account_name", null)
|
|
113
|
+
.set("account_key", AZURE_ACCOUNT_KEY)
|
|
114
|
+
.set("container", AZURE_CONTAINER)
|
|
115
|
+
.set("path_prefix", AZURE_PATH_PREFIX)
|
|
116
|
+
.set("last_path", "")
|
|
117
|
+
.set("file_ext", ".csv")
|
|
118
|
+
.set("parser", parserConfig(schemaConfig()));
|
|
119
|
+
|
|
120
|
+
runner.transaction(config, new Control());
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
@Test(expected = ConfigException.class)
|
|
124
|
+
public void checkDefaultValuesAccountKeyIsNull()
|
|
125
|
+
{
|
|
126
|
+
ConfigSource config = Exec.newConfigSource()
|
|
127
|
+
.set("account_name", AZURE_ACCOUNT_NAME)
|
|
128
|
+
.set("account_key", null)
|
|
129
|
+
.set("container", AZURE_CONTAINER)
|
|
130
|
+
.set("path_prefix", AZURE_PATH_PREFIX)
|
|
131
|
+
.set("last_path", "")
|
|
132
|
+
.set("file_ext", ".csv")
|
|
133
|
+
.set("parser", parserConfig(schemaConfig()));
|
|
134
|
+
|
|
135
|
+
runner.transaction(config, new Control());
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
@Test(expected = ConfigException.class)
|
|
139
|
+
public void checkDefaultValuesContainerIsNull()
|
|
140
|
+
{
|
|
141
|
+
ConfigSource config = Exec.newConfigSource()
|
|
142
|
+
.set("account_name", AZURE_ACCOUNT_NAME)
|
|
143
|
+
.set("account_key", AZURE_ACCOUNT_KEY)
|
|
144
|
+
.set("container", null)
|
|
145
|
+
.set("path_prefix", AZURE_PATH_PREFIX)
|
|
146
|
+
.set("last_path", "")
|
|
147
|
+
.set("file_ext", ".csv")
|
|
148
|
+
.set("parser", parserConfig(schemaConfig()));
|
|
149
|
+
|
|
150
|
+
runner.transaction(config, new Control());
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
@Test
|
|
154
|
+
public void testAzureClientCreateSuccessfully()
|
|
155
|
+
throws GeneralSecurityException, IOException, NoSuchMethodException,
|
|
156
|
+
IllegalAccessException, InvocationTargetException
|
|
157
|
+
{
|
|
158
|
+
PluginTask task = config().loadConfig(PluginTask.class);
|
|
159
|
+
|
|
160
|
+
Method method = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
|
|
161
|
+
method.setAccessible(true);
|
|
162
|
+
method.invoke(plugin, task.getAccountName(), task.getAccountKey()); // no errors happens
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
@Test
|
|
166
|
+
public void testResume()
|
|
167
|
+
{
|
|
168
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
169
|
+
task.setFiles(Arrays.asList("in/aa/a"));
|
|
170
|
+
ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
|
|
171
|
+
{
|
|
172
|
+
@Override
|
|
173
|
+
public List<TaskReport> run(TaskSource taskSource, int taskCount)
|
|
174
|
+
{
|
|
175
|
+
return emptyTaskReports(taskCount);
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
@Test
|
|
182
|
+
public void testCleanup()
|
|
183
|
+
{
|
|
184
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
185
|
+
plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
@Test
|
|
189
|
+
public void testListFiles()
|
|
190
|
+
throws NoSuchMethodException, IllegalAccessException, InvocationTargetException
|
|
191
|
+
{
|
|
192
|
+
List<String> expected = Arrays.asList(
|
|
193
|
+
AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_01.csv",
|
|
194
|
+
AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_02.csv"
|
|
195
|
+
);
|
|
196
|
+
|
|
197
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
198
|
+
ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
|
199
|
+
@Override
|
|
200
|
+
public List<TaskReport> run(TaskSource taskSource, int taskCount)
|
|
201
|
+
{
|
|
202
|
+
assertEquals(2, taskCount);
|
|
203
|
+
return emptyTaskReports(taskCount);
|
|
204
|
+
}
|
|
205
|
+
});
|
|
206
|
+
|
|
207
|
+
Method newAzureClient = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
|
|
208
|
+
newAzureClient.setAccessible(true);
|
|
209
|
+
CloudBlobClient client = (CloudBlobClient) newAzureClient.invoke(plugin, task.getAccountName(), task.getAccountKey());
|
|
210
|
+
|
|
211
|
+
Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", CloudBlobClient.class, PluginTask.class);
|
|
212
|
+
listFiles.setAccessible(true);
|
|
213
|
+
List<String> actual = (List<String>) listFiles.invoke(plugin, client, task);
|
|
214
|
+
assertEquals(expected, actual);
|
|
215
|
+
assertEquals(AZURE_CONTAINER_IMPORT_DIRECTORY + "sample_02.csv", configDiff.get(String.class, "last_path"));
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
@Test
|
|
219
|
+
public void testAzureFileInputByOpen()
|
|
220
|
+
throws NoSuchMethodException, IllegalAccessException, InvocationTargetException, IOException
|
|
221
|
+
{
|
|
222
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
223
|
+
runner.transaction(config, new Control());
|
|
224
|
+
|
|
225
|
+
Method newAzureClient = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("newAzureClient", String.class, String.class);
|
|
226
|
+
newAzureClient.setAccessible(true);
|
|
227
|
+
CloudBlobClient client = (CloudBlobClient) newAzureClient.invoke(plugin, task.getAccountName(), task.getAccountKey());
|
|
228
|
+
|
|
229
|
+
Method listFiles = AzureBlobStorageFileInputPlugin.class.getDeclaredMethod("listFiles", CloudBlobClient.class, PluginTask.class);
|
|
230
|
+
listFiles.setAccessible(true);
|
|
231
|
+
task.setFiles((List<String>) listFiles.invoke(plugin, client, task));
|
|
232
|
+
|
|
233
|
+
assertRecords(config, output);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
static List<TaskReport> emptyTaskReports(int taskCount)
|
|
237
|
+
{
|
|
238
|
+
ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
|
239
|
+
for (int i = 0; i < taskCount; i++) {
|
|
240
|
+
reports.add(Exec.newTaskReport());
|
|
241
|
+
}
|
|
242
|
+
return reports.build();
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
private class Control
|
|
246
|
+
implements InputPlugin.Control
|
|
247
|
+
{
|
|
248
|
+
@Override
|
|
249
|
+
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
|
250
|
+
{
|
|
251
|
+
List<TaskReport> reports = new ArrayList<>();
|
|
252
|
+
for (int i = 0; i < taskCount; i++) {
|
|
253
|
+
reports.add(runner.run(taskSource, schema, i, output));
|
|
254
|
+
}
|
|
255
|
+
return reports;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
|
|
260
|
+
{
|
|
261
|
+
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
|
262
|
+
builder.put("type", "csv");
|
|
263
|
+
builder.put("newline", "CRLF");
|
|
264
|
+
builder.put("delimiter", ",");
|
|
265
|
+
builder.put("quote", "\"");
|
|
266
|
+
builder.put("escape", "\"");
|
|
267
|
+
builder.put("trim_if_not_quoted", false);
|
|
268
|
+
builder.put("skip_header_lines", 1);
|
|
269
|
+
builder.put("allow_extra_columns", false);
|
|
270
|
+
builder.put("allow_optional_columns", false);
|
|
271
|
+
builder.put("columns", schemaConfig);
|
|
272
|
+
return builder.build();
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
private ImmutableList<Object> schemaConfig()
|
|
276
|
+
{
|
|
277
|
+
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
|
278
|
+
builder.add(ImmutableMap.of("name", "id", "type", "long"));
|
|
279
|
+
builder.add(ImmutableMap.of("name", "account", "type", "long"));
|
|
280
|
+
builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
|
|
281
|
+
builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
|
|
282
|
+
builder.add(ImmutableMap.of("name", "comment", "type", "string"));
|
|
283
|
+
builder.add(ImmutableMap.of("name", "json_column", "type", "json"));
|
|
284
|
+
return builder.build();
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
private void assertRecords(ConfigSource config, MockPageOutput output)
|
|
288
|
+
{
|
|
289
|
+
List<Object[]> records = getRecords(config, output);
|
|
290
|
+
assertEquals(10, records.size());
|
|
291
|
+
{
|
|
292
|
+
Object[] record = records.get(0);
|
|
293
|
+
assertEquals(1L, record[0]);
|
|
294
|
+
assertEquals(32864L, record[1]);
|
|
295
|
+
assertEquals("2015-01-27 19:23:49 UTC", record[2].toString());
|
|
296
|
+
assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
|
|
297
|
+
assertEquals("embulk", record[4]);
|
|
298
|
+
assertEquals("{\"k\":true}", record[5].toString());
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
{
|
|
302
|
+
Object[] record = records.get(1);
|
|
303
|
+
assertEquals(2L, record[0]);
|
|
304
|
+
assertEquals(14824L, record[1]);
|
|
305
|
+
assertEquals("2015-01-27 19:01:23 UTC", record[2].toString());
|
|
306
|
+
assertEquals("2015-01-27 00:00:00 UTC", record[3].toString());
|
|
307
|
+
assertEquals("embulk jruby", record[4]);
|
|
308
|
+
assertEquals("{\"k\":1}", record[5].toString());
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
{
|
|
312
|
+
Object[] record = records.get(2);
|
|
313
|
+
assertEquals("{\"k\":1.23}", record[5].toString());
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
{
|
|
317
|
+
Object[] record = records.get(3);
|
|
318
|
+
assertEquals("{\"k\":\"v\"}", record[5].toString());
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
{
|
|
322
|
+
Object[] record = records.get(4);
|
|
323
|
+
assertEquals("{\"k\":\"2015-02-03 08:13:45\"}", record[5].toString());
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
|
|
328
|
+
{
|
|
329
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
|
330
|
+
return Pages.toObjects(schema, output.pages);
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
private static String getDirectory(String dir)
|
|
334
|
+
{
|
|
335
|
+
if (!dir.isEmpty() && !dir.endsWith("/")) {
|
|
336
|
+
dir = dir + "/";
|
|
337
|
+
}
|
|
338
|
+
if (dir.startsWith("/")) {
|
|
339
|
+
dir = dir.replaceFirst("/", "");
|
|
340
|
+
}
|
|
341
|
+
return dir;
|
|
342
|
+
}
|
|
5
343
|
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
id,account,time,purchase,comment,json_column
|
|
2
|
+
1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
|
|
3
|
+
2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
|
|
4
|
+
3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
|
|
5
|
+
4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
|
|
6
|
+
5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
id,account,time,purchase,comment,json_column
|
|
2
|
+
1,32864,2015-01-27 19:23:49,20150127,embulk,{"k":true}
|
|
3
|
+
2,14824,2015-01-27 19:01:23,20150127,embulk jruby,{"k":1}
|
|
4
|
+
3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin",{"k":1.23}
|
|
5
|
+
4,11270,2015-01-29 11:54:36,20150129,NULL,{"k":"v"}
|
|
6
|
+
5,53231,2015-01-30 13:48:12,20150130,NULL,{"k":"2015-02-03 08:13:45"}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-input-azure_blob_storage
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Satoshi Akama
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2016-
|
|
11
|
+
date: 2016-03-16 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -46,9 +46,11 @@ extensions: []
|
|
|
46
46
|
extra_rdoc_files: []
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
|
-
-
|
|
49
|
+
- CHANGELOG.md
|
|
50
50
|
- README.md
|
|
51
51
|
- build.gradle
|
|
52
|
+
- config/checkstyle/checkstyle.xml
|
|
53
|
+
- config/checkstyle/default.xml
|
|
52
54
|
- gradle/wrapper/gradle-wrapper.jar
|
|
53
55
|
- gradle/wrapper/gradle-wrapper.properties
|
|
54
56
|
- gradlew
|
|
@@ -56,9 +58,11 @@ files:
|
|
|
56
58
|
- lib/embulk/input/azure_blob_storage.rb
|
|
57
59
|
- src/main/java/org/embulk/input/azure_blob_storage/AzureBlobStorageFileInputPlugin.java
|
|
58
60
|
- src/test/java/org/embulk/input/azure_blob_storage/TestAzureBlobStorageFileInputPlugin.java
|
|
61
|
+
- src/test/resources/sample_01.csv
|
|
62
|
+
- src/test/resources/sample_02.csv
|
|
59
63
|
- classpath/azure-storage-4.0.0.jar
|
|
60
64
|
- classpath/commons-lang3-3.4.jar
|
|
61
|
-
- classpath/embulk-input-azure_blob_storage-0.1.
|
|
65
|
+
- classpath/embulk-input-azure_blob_storage-0.1.3.jar
|
|
62
66
|
- classpath/jackson-core-2.6.0.jar
|
|
63
67
|
homepage: https://github.com/sakama/embulk-input-azure_blob_storage
|
|
64
68
|
licenses:
|
data/ChangeLog
DELETED
|
Binary file
|