embulk-output-sftp 0.0.4 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +66 -2
- data/build.gradle +26 -6
- data/classpath/embulk-output-sftp-0.0.5.jar +0 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/sftp/ProxyTask.java +85 -0
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +62 -12
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +6 -1
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +173 -31
- metadata +7 -3
- data/classpath/embulk-output-sftp-0.0.4.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 865291280d7c6936d0e4f12655a00b854f880153
|
|
4
|
+
data.tar.gz: dfe086b9210d16e7da2e21d5f77e42c38bd7f23b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b226b3a68c6e81ddf17ff22577ffd80cae2b58201b18a38bae681b0729e531769426e3f760694d36121e1885a6dfc21383a91c4fe5664203aafb58ad8c00c0fd
|
|
7
|
+
data.tar.gz: 6bdda04f1ba31689656896323098be6e43e34c88cec3ea7922c6e0c55c6d8bc3b593f89ea863fcefb8a537da2ae38e6a2ebf159008e44e5d475cc025014fe80d
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
0.0.5 (2016-03-09)
|
|
2
|
+
==================
|
|
3
|
+
- Add: Support MapReduce executor
|
|
4
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/8
|
|
5
|
+
- Fix: Use ConfigException instead of RuntimeException
|
|
6
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/9/files
|
|
7
|
+
- Fix: Check to exist parent directory before uploading files
|
|
8
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/13
|
|
9
|
+
- Add: Support proxy settings
|
|
10
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/11
|
data/README.md
CHANGED
|
@@ -14,10 +14,10 @@ Stores files on a SFTP Server
|
|
|
14
14
|
## Configuration
|
|
15
15
|
|
|
16
16
|
- **host**: (string, required)
|
|
17
|
-
- **port**: (
|
|
17
|
+
- **port**: (int, default: `22`)
|
|
18
18
|
- **user**: (string, required)
|
|
19
19
|
- **password**: (string, default: `null`)
|
|
20
|
-
- **secret_key_file**: (string, default: `null`)
|
|
20
|
+
- **secret_key_file**: (string, default: `null`) [see below](#secret-keyfile-configuration)
|
|
21
21
|
- **secret_key_passphrase**: (string, default: `""`)
|
|
22
22
|
- **user_directory_is_root**: (boolean, default: `true`)
|
|
23
23
|
- **timeout**: sftp connection timeout seconds (integer, default: `600`)
|
|
@@ -25,6 +25,19 @@ Stores files on a SFTP Server
|
|
|
25
25
|
- **file_ext**: Extension of output files (string, required)
|
|
26
26
|
- **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
|
|
27
27
|
|
|
28
|
+
### Proxy configuration
|
|
29
|
+
|
|
30
|
+
- **proxy**:
|
|
31
|
+
- **type**: (string(http | socks | stream), required, default: `null`)
|
|
32
|
+
- **http**: use HTTP Proxy
|
|
33
|
+
- **socks**: use SOCKS Proxy
|
|
34
|
+
- **stream**: Connects to the SFTP server through a remote host reached by SSH
|
|
35
|
+
- **host**: (string, required)
|
|
36
|
+
- **port**: (int, default: `22`)
|
|
37
|
+
- **user**: (string, optional)
|
|
38
|
+
- **password**: (string, optional, default: `null`)
|
|
39
|
+
- **command**: (string, optional)
|
|
40
|
+
|
|
28
41
|
## Example
|
|
29
42
|
|
|
30
43
|
```yaml
|
|
@@ -42,6 +55,53 @@ out:
|
|
|
42
55
|
sequence_format: ".%01d%01d"
|
|
43
56
|
```
|
|
44
57
|
|
|
58
|
+
With proxy
|
|
59
|
+
```yaml
|
|
60
|
+
out:
|
|
61
|
+
type: sftp
|
|
62
|
+
host: 127.0.0.1
|
|
63
|
+
port: 22
|
|
64
|
+
user: embulk
|
|
65
|
+
secret_key_file: /Users/embulk/.ssh/id_rsa
|
|
66
|
+
secret_key_passphrase: secret_pass
|
|
67
|
+
user_directory_is_root: false
|
|
68
|
+
timeout: 600
|
|
69
|
+
path_prefix: /data/sftp
|
|
70
|
+
proxy:
|
|
71
|
+
type: http
|
|
72
|
+
host: proxy_host
|
|
73
|
+
port: 8080
|
|
74
|
+
user: proxy_user
|
|
75
|
+
password: proxy_secret_pass
|
|
76
|
+
command:
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Secret Keyfile configuration
|
|
80
|
+
|
|
81
|
+
Please set path of secret_key_file as follows.
|
|
82
|
+
```yaml
|
|
83
|
+
out:
|
|
84
|
+
type: sftp
|
|
85
|
+
...
|
|
86
|
+
secret_key_file: /path/to/id_rsa
|
|
87
|
+
...
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
You can also embed contents of secret_key_file at config.yml.
|
|
91
|
+
```yaml
|
|
92
|
+
out:
|
|
93
|
+
type: sftp
|
|
94
|
+
...
|
|
95
|
+
secret_key_file:
|
|
96
|
+
content |
|
|
97
|
+
-----BEGIN RSA PRIVATE KEY-----
|
|
98
|
+
ABCDEFG...
|
|
99
|
+
HIJKLMN...
|
|
100
|
+
OPQRSTU...
|
|
101
|
+
-----END RSA PRIVATE KEY-----
|
|
102
|
+
...
|
|
103
|
+
```
|
|
104
|
+
|
|
45
105
|
## Run Example
|
|
46
106
|
replace settings in `example/sample.yml` before running.
|
|
47
107
|
|
|
@@ -59,3 +119,7 @@ $ ./gradlew gem # -t to watch change of files and rebuild continuously
|
|
|
59
119
|
## Note
|
|
60
120
|
|
|
61
121
|
This plugin uses "org.apache.commons:commons-vfs" and the library uses the logger "org.apache.commons.logging.Log". So, this plugin suppress the logger's message except when embulk log level is debug.
|
|
122
|
+
|
|
123
|
+
## Contributors
|
|
124
|
+
- Satoshi Akama (@sakama)
|
|
125
|
+
- Rudolph Miller (@Rudolph-Miller)
|
data/build.gradle
CHANGED
|
@@ -4,6 +4,7 @@ plugins {
|
|
|
4
4
|
id "com.github.kt3k.coveralls" version "2.4.0"
|
|
5
5
|
id "jacoco"
|
|
6
6
|
id "java"
|
|
7
|
+
id "checkstyle"
|
|
7
8
|
}
|
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
|
9
10
|
repositories {
|
|
@@ -14,20 +15,22 @@ configurations {
|
|
|
14
15
|
provided
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
version = "0.0.
|
|
18
|
+
version = "0.0.5"
|
|
18
19
|
sourceCompatibility = 1.7
|
|
19
20
|
targetCompatibility = 1.7
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.6"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.6"
|
|
24
25
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
|
25
26
|
compile "org.apache.commons:commons-vfs2:2.+"
|
|
26
27
|
compile "com.jcraft:jsch:0.1.53"
|
|
27
28
|
testCompile "junit:junit:4.+"
|
|
28
|
-
testCompile "org.embulk:embulk-core:0.
|
|
29
|
-
testCompile "org.embulk:embulk-standards:0.
|
|
30
|
-
testCompile "org.apache.sshd:apache-sshd:1
|
|
29
|
+
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
|
30
|
+
testCompile "org.embulk:embulk-standards:0.8.6"
|
|
31
|
+
testCompile "org.apache.sshd:apache-sshd:1.1.0+"
|
|
32
|
+
testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
|
|
33
|
+
testCompile "io.netty:netty-all:4.0.34.Final"
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
jacocoTestReport {
|
|
@@ -44,6 +47,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
44
47
|
}
|
|
45
48
|
clean { delete "classpath" }
|
|
46
49
|
|
|
50
|
+
checkstyle {
|
|
51
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
52
|
+
toolVersion = '6.14.1'
|
|
53
|
+
}
|
|
54
|
+
checkstyleMain {
|
|
55
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
56
|
+
ignoreFailures = true
|
|
57
|
+
}
|
|
58
|
+
checkstyleTest {
|
|
59
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
60
|
+
ignoreFailures = true
|
|
61
|
+
}
|
|
62
|
+
task checkstyle(type: Checkstyle) {
|
|
63
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
64
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
65
|
+
}
|
|
66
|
+
|
|
47
67
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
48
68
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
49
69
|
script "${project.name}.gemspec"
|
|
Binary file
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
package org.embulk.output.sftp;
|
|
2
|
+
|
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
|
4
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
|
5
|
+
import com.google.common.base.Optional;
|
|
6
|
+
import org.apache.commons.vfs2.FileSystemOptions;
|
|
7
|
+
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
|
8
|
+
import org.embulk.config.Config;
|
|
9
|
+
import org.embulk.config.ConfigDefault;
|
|
10
|
+
import org.embulk.config.ConfigException;
|
|
11
|
+
import org.embulk.config.Task;
|
|
12
|
+
|
|
13
|
+
import java.util.Locale;
|
|
14
|
+
|
|
15
|
+
interface ProxyTask
|
|
16
|
+
extends Task
|
|
17
|
+
{
|
|
18
|
+
@Config("type")
|
|
19
|
+
public ProxyType getType();
|
|
20
|
+
|
|
21
|
+
@Config("host")
|
|
22
|
+
public Optional<String> getHost();
|
|
23
|
+
|
|
24
|
+
@Config("user")
|
|
25
|
+
@ConfigDefault("null")
|
|
26
|
+
public Optional<String> getUser();
|
|
27
|
+
|
|
28
|
+
@Config("password")
|
|
29
|
+
@ConfigDefault("null")
|
|
30
|
+
public Optional<String> getPassword();
|
|
31
|
+
|
|
32
|
+
@Config("port")
|
|
33
|
+
@ConfigDefault("22")
|
|
34
|
+
public int getPort();
|
|
35
|
+
|
|
36
|
+
@Config("command")
|
|
37
|
+
@ConfigDefault("null")
|
|
38
|
+
public Optional<String> getCommand();
|
|
39
|
+
|
|
40
|
+
public enum ProxyType
|
|
41
|
+
{
|
|
42
|
+
HTTP,
|
|
43
|
+
SOCKS,
|
|
44
|
+
STREAM;
|
|
45
|
+
|
|
46
|
+
@JsonValue
|
|
47
|
+
@Override
|
|
48
|
+
public String toString()
|
|
49
|
+
{
|
|
50
|
+
return name().toLowerCase(Locale.ENGLISH);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@JsonCreator
|
|
54
|
+
public static ProxyType fromString(String value)
|
|
55
|
+
{
|
|
56
|
+
switch (value) {
|
|
57
|
+
case "http":
|
|
58
|
+
return HTTP;
|
|
59
|
+
case "socks":
|
|
60
|
+
return SOCKS;
|
|
61
|
+
case "stream":
|
|
62
|
+
return STREAM;
|
|
63
|
+
default:
|
|
64
|
+
throw new ConfigException(String.format("Unknown proxy type '%s'. Supported proxy types are http, socks, stream", value));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public static SftpFileSystemConfigBuilder setProxyType(SftpFileSystemConfigBuilder builder, FileSystemOptions fsOptions, ProxyTask.ProxyType type)
|
|
69
|
+
{
|
|
70
|
+
SftpFileSystemConfigBuilder.ProxyType setType = null;
|
|
71
|
+
switch (type) {
|
|
72
|
+
case HTTP:
|
|
73
|
+
setType = SftpFileSystemConfigBuilder.PROXY_HTTP;
|
|
74
|
+
break;
|
|
75
|
+
case SOCKS:
|
|
76
|
+
setType = SftpFileSystemConfigBuilder.PROXY_SOCKS5;
|
|
77
|
+
break;
|
|
78
|
+
case STREAM:
|
|
79
|
+
setType = SftpFileSystemConfigBuilder.PROXY_STREAM;
|
|
80
|
+
}
|
|
81
|
+
builder.setProxyType(fsOptions, setType);
|
|
82
|
+
return builder;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
package org.embulk.output.sftp;
|
|
2
2
|
|
|
3
|
+
import com.google.common.base.Function;
|
|
3
4
|
import com.google.common.base.Throwables;
|
|
4
5
|
import org.apache.commons.vfs2.FileObject;
|
|
5
6
|
import org.apache.commons.vfs2.FileSystemException;
|
|
@@ -7,11 +8,13 @@ import org.apache.commons.vfs2.FileSystemOptions;
|
|
|
7
8
|
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
|
8
9
|
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
|
9
10
|
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
|
11
|
+
import org.embulk.config.ConfigException;
|
|
10
12
|
import org.embulk.config.TaskReport;
|
|
11
13
|
import org.embulk.spi.Buffer;
|
|
12
14
|
import org.embulk.spi.Exec;
|
|
13
15
|
import org.embulk.spi.FileOutput;
|
|
14
16
|
import org.embulk.spi.TransactionalFileOutput;
|
|
17
|
+
import org.embulk.spi.unit.LocalFile;
|
|
15
18
|
import org.slf4j.Logger;
|
|
16
19
|
|
|
17
20
|
import java.io.File;
|
|
@@ -56,7 +59,7 @@ public class SftpFileOutput
|
|
|
56
59
|
}
|
|
57
60
|
catch (FileSystemException e) {
|
|
58
61
|
logger.error(e.getMessage());
|
|
59
|
-
throw new
|
|
62
|
+
throw new ConfigException(e);
|
|
60
63
|
}
|
|
61
64
|
|
|
62
65
|
return manager;
|
|
@@ -76,18 +79,45 @@ public class SftpFileOutput
|
|
|
76
79
|
FileSystemOptions fsOptions = new FileSystemOptions();
|
|
77
80
|
|
|
78
81
|
try {
|
|
79
|
-
SftpFileSystemConfigBuilder.getInstance()
|
|
80
|
-
|
|
81
|
-
|
|
82
|
+
SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
|
|
83
|
+
builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
|
|
84
|
+
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout());
|
|
85
|
+
builder.setStrictHostKeyChecking(fsOptions, "no");
|
|
82
86
|
if (task.getSecretKeyFilePath().isPresent()) {
|
|
83
|
-
IdentityInfo identityInfo = new IdentityInfo(
|
|
84
|
-
|
|
87
|
+
IdentityInfo identityInfo = new IdentityInfo(
|
|
88
|
+
new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
|
|
89
|
+
task.getSecretKeyPassphrase().getBytes()
|
|
90
|
+
);
|
|
91
|
+
builder.setIdentityInfo(fsOptions, identityInfo);
|
|
85
92
|
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
|
86
93
|
}
|
|
94
|
+
|
|
95
|
+
if (task.getProxy().isPresent()) {
|
|
96
|
+
ProxyTask proxy = task.getProxy().get();
|
|
97
|
+
|
|
98
|
+
ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
|
|
99
|
+
|
|
100
|
+
if (proxy.getHost().isPresent()) {
|
|
101
|
+
builder.setProxyHost(fsOptions, proxy.getHost().get());
|
|
102
|
+
builder.setProxyPort(fsOptions, proxy.getPort());
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (proxy.getUser().isPresent()) {
|
|
106
|
+
builder.setProxyUser(fsOptions, proxy.getUser().get());
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (proxy.getPassword().isPresent()) {
|
|
110
|
+
builder.setProxyPassword(fsOptions, proxy.getPassword().get());
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (proxy.getCommand().isPresent()) {
|
|
114
|
+
builder.setProxyCommand(fsOptions, proxy.getCommand().get());
|
|
115
|
+
}
|
|
116
|
+
}
|
|
87
117
|
}
|
|
88
118
|
catch (FileSystemException e) {
|
|
89
119
|
logger.error(e.getMessage());
|
|
90
|
-
throw new
|
|
120
|
+
throw new ConfigException(e);
|
|
91
121
|
}
|
|
92
122
|
|
|
93
123
|
return fsOptions;
|
|
@@ -137,7 +167,9 @@ public class SftpFileOutput
|
|
|
137
167
|
logger.error(e.getMessage());
|
|
138
168
|
Throwables.propagate(e);
|
|
139
169
|
}
|
|
140
|
-
|
|
170
|
+
finally {
|
|
171
|
+
buffer.release();
|
|
172
|
+
}
|
|
141
173
|
}
|
|
142
174
|
|
|
143
175
|
@Override
|
|
@@ -161,10 +193,9 @@ public class SftpFileOutput
|
|
|
161
193
|
@Override
|
|
162
194
|
public TaskReport commit()
|
|
163
195
|
{
|
|
164
|
-
return
|
|
196
|
+
return Exec.newTaskReport();
|
|
165
197
|
}
|
|
166
198
|
|
|
167
|
-
|
|
168
199
|
private void closeCurrentFile()
|
|
169
200
|
{
|
|
170
201
|
if (currentFile == null) {
|
|
@@ -194,7 +225,7 @@ public class SftpFileOutput
|
|
|
194
225
|
}
|
|
195
226
|
catch (URISyntaxException e) {
|
|
196
227
|
logger.error(e.getMessage());
|
|
197
|
-
throw new
|
|
228
|
+
throw new ConfigException(e);
|
|
198
229
|
}
|
|
199
230
|
}
|
|
200
231
|
|
|
@@ -209,7 +240,15 @@ public class SftpFileOutput
|
|
|
209
240
|
int count = 0;
|
|
210
241
|
while (true) {
|
|
211
242
|
try {
|
|
212
|
-
|
|
243
|
+
FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
|
|
244
|
+
if (file.getParent().exists()) {
|
|
245
|
+
logger.info("parent directory {} exists there", file.getParent());
|
|
246
|
+
return file;
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
logger.info("trying to create parent directory {}", file.getParent());
|
|
250
|
+
file.getParent().createFolder();
|
|
251
|
+
}
|
|
213
252
|
}
|
|
214
253
|
catch (FileSystemException e) {
|
|
215
254
|
if (++count == maxConnectionRetry) {
|
|
@@ -230,4 +269,15 @@ public class SftpFileOutput
|
|
|
230
269
|
}
|
|
231
270
|
}
|
|
232
271
|
}
|
|
272
|
+
|
|
273
|
+
private Function<LocalFile, String> localFileToPathString()
|
|
274
|
+
{
|
|
275
|
+
return new Function<LocalFile, String>()
|
|
276
|
+
{
|
|
277
|
+
public String apply(LocalFile file)
|
|
278
|
+
{
|
|
279
|
+
return file.getPath().toString();
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
}
|
|
233
283
|
}
|
|
@@ -11,6 +11,7 @@ import org.embulk.config.TaskSource;
|
|
|
11
11
|
import org.embulk.spi.Exec;
|
|
12
12
|
import org.embulk.spi.FileOutputPlugin;
|
|
13
13
|
import org.embulk.spi.TransactionalFileOutput;
|
|
14
|
+
import org.embulk.spi.unit.LocalFile;
|
|
14
15
|
import org.slf4j.Logger;
|
|
15
16
|
|
|
16
17
|
import java.util.List;
|
|
@@ -39,7 +40,8 @@ public class SftpFileOutputPlugin
|
|
|
39
40
|
|
|
40
41
|
@Config("secret_key_file")
|
|
41
42
|
@ConfigDefault("null")
|
|
42
|
-
public Optional<
|
|
43
|
+
public Optional<LocalFile> getSecretKeyFilePath();
|
|
44
|
+
public void setSecretKeyFilePath(Optional<LocalFile> secretKeyFilePath);
|
|
43
45
|
|
|
44
46
|
@Config("secret_key_passphrase")
|
|
45
47
|
@ConfigDefault("\"\"")
|
|
@@ -67,6 +69,9 @@ public class SftpFileOutputPlugin
|
|
|
67
69
|
@ConfigDefault("\"%03d.%02d.\"")
|
|
68
70
|
public String getSequenceFormat();
|
|
69
71
|
|
|
72
|
+
@Config("proxy")
|
|
73
|
+
@ConfigDefault("null")
|
|
74
|
+
public Optional<ProxyTask> getProxy();
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
@Override
|
|
@@ -2,8 +2,6 @@ package org.embulk.output.sftp;
|
|
|
2
2
|
|
|
3
3
|
import com.google.common.base.Charsets;
|
|
4
4
|
import com.google.common.base.Optional;
|
|
5
|
-
import com.google.common.base.Splitter;
|
|
6
|
-
import com.google.common.base.Throwables;
|
|
7
5
|
import com.google.common.collect.Lists;
|
|
8
6
|
import com.google.common.io.Resources;
|
|
9
7
|
import org.apache.commons.vfs2.FileSystemException;
|
|
@@ -13,11 +11,12 @@ import org.apache.sshd.server.Command;
|
|
|
13
11
|
import org.apache.sshd.server.SshServer;
|
|
14
12
|
import org.apache.sshd.server.auth.password.PasswordAuthenticator;
|
|
15
13
|
import org.apache.sshd.server.auth.pubkey.PublickeyAuthenticator;
|
|
16
|
-
import org.apache.sshd.server.command.ScpCommandFactory;
|
|
17
14
|
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
|
|
15
|
+
import org.apache.sshd.server.scp.ScpCommandFactory;
|
|
18
16
|
import org.apache.sshd.server.session.ServerSession;
|
|
19
17
|
import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory;
|
|
20
18
|
import org.embulk.EmbulkTestRuntime;
|
|
19
|
+
import org.embulk.config.ConfigException;
|
|
21
20
|
import org.embulk.config.ConfigLoader;
|
|
22
21
|
import org.embulk.config.ConfigSource;
|
|
23
22
|
import org.embulk.config.TaskReport;
|
|
@@ -32,37 +31,39 @@ import org.embulk.spi.Schema;
|
|
|
32
31
|
import org.embulk.spi.TransactionalPageOutput;
|
|
33
32
|
import org.embulk.spi.time.Timestamp;
|
|
34
33
|
import org.hamcrest.CoreMatchers;
|
|
35
|
-
import org.hamcrest.Matcher;
|
|
36
34
|
import org.junit.After;
|
|
37
35
|
import org.junit.Before;
|
|
38
36
|
import org.junit.Rule;
|
|
39
37
|
import org.junit.Test;
|
|
40
38
|
import org.junit.rules.ExpectedException;
|
|
41
39
|
import org.junit.rules.TemporaryFolder;
|
|
40
|
+
import org.littleshoot.proxy.HttpProxyServer;
|
|
41
|
+
import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
|
|
42
42
|
import org.slf4j.Logger;
|
|
43
43
|
|
|
44
44
|
import java.io.File;
|
|
45
45
|
import java.io.IOException;
|
|
46
46
|
import java.nio.file.DirectoryStream;
|
|
47
|
-
import java.nio.file.FileSystems;
|
|
48
47
|
import java.nio.file.Files;
|
|
49
48
|
import java.nio.file.Path;
|
|
50
|
-
import java.nio.file.PathMatcher;
|
|
51
49
|
import java.nio.file.Paths;
|
|
52
50
|
import java.security.PublicKey;
|
|
53
|
-
import java.util.Arrays;
|
|
54
51
|
import java.util.Collections;
|
|
55
52
|
import java.util.List;
|
|
56
53
|
|
|
57
54
|
import static com.google.common.io.Files.readLines;
|
|
58
|
-
import static org.embulk.spi.type.Types
|
|
55
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
56
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
|
57
|
+
import static org.embulk.spi.type.Types.JSON;
|
|
58
|
+
import static org.embulk.spi.type.Types.LONG;
|
|
59
|
+
import static org.embulk.spi.type.Types.STRING;
|
|
60
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
59
61
|
import static org.hamcrest.CoreMatchers.containsString;
|
|
60
62
|
import static org.hamcrest.CoreMatchers.hasItem;
|
|
61
|
-
import static org.hamcrest.CoreMatchers.instanceOf;
|
|
62
|
-
import static org.hamcrest.MatcherAssert.*;
|
|
63
|
-
import static org.hamcrest.core.Is.is;
|
|
64
63
|
import static org.junit.Assert.assertEquals;
|
|
65
|
-
|
|
64
|
+
import static org.junit.Assert.assertThat;
|
|
65
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
|
66
|
+
import static org.msgpack.value.ValueFactory.newString;
|
|
66
67
|
|
|
67
68
|
public class TestSftpFileOutputPlugin
|
|
68
69
|
{
|
|
@@ -80,6 +81,8 @@ public class TestSftpFileOutputPlugin
|
|
|
80
81
|
private SshServer sshServer;
|
|
81
82
|
private static final String HOST = "127.0.0.1";
|
|
82
83
|
private static final int PORT = 20022;
|
|
84
|
+
private static final String PROXY_HOST = "127.0.0.1";
|
|
85
|
+
private static final int PROXY_PORT = 8080;
|
|
83
86
|
private static final String USERNAME = "username";
|
|
84
87
|
private static final String PASSWORD = "password";
|
|
85
88
|
private static final String SECRET_KEY_FILE = Resources.getResource("id_rsa").getPath();
|
|
@@ -90,6 +93,7 @@ public class TestSftpFileOutputPlugin
|
|
|
90
93
|
.add("_c2", DOUBLE)
|
|
91
94
|
.add("_c3", STRING)
|
|
92
95
|
.add("_c4", TIMESTAMP)
|
|
96
|
+
.add("_c5", JSON)
|
|
93
97
|
.build();
|
|
94
98
|
|
|
95
99
|
@Before
|
|
@@ -100,13 +104,18 @@ public class TestSftpFileOutputPlugin
|
|
|
100
104
|
SftpFileOutputPlugin sftpFileOutputPlugin = new SftpFileOutputPlugin();
|
|
101
105
|
runner = new FileOutputRunner(sftpFileOutputPlugin);
|
|
102
106
|
|
|
107
|
+
sshServer = createSshServer(HOST, PORT, USERNAME, PASSWORD);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
private SshServer createSshServer(String host, int port, final String sshUsername, final String sshPassword)
|
|
111
|
+
{
|
|
103
112
|
// setup a mock sftp server
|
|
104
|
-
sshServer = SshServer.setUpDefaultServer();
|
|
113
|
+
SshServer sshServer = SshServer.setUpDefaultServer();
|
|
105
114
|
VirtualFileSystemFactory fsFactory = new VirtualFileSystemFactory();
|
|
106
|
-
fsFactory.setUserHomeDir(
|
|
115
|
+
fsFactory.setUserHomeDir(sshUsername, testFolder.getRoot().toPath());
|
|
107
116
|
sshServer.setFileSystemFactory(fsFactory);
|
|
108
|
-
sshServer.setHost(
|
|
109
|
-
sshServer.setPort(
|
|
117
|
+
sshServer.setHost(host);
|
|
118
|
+
sshServer.setPort(port);
|
|
110
119
|
sshServer.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystemFactory()));
|
|
111
120
|
sshServer.setCommandFactory(new ScpCommandFactory());
|
|
112
121
|
sshServer.setKeyPairProvider(new SimpleGeneratorHostKeyProvider());
|
|
@@ -115,7 +124,7 @@ public class TestSftpFileOutputPlugin
|
|
|
115
124
|
@Override
|
|
116
125
|
public boolean authenticate(final String username, final String password, final ServerSession session)
|
|
117
126
|
{
|
|
118
|
-
return
|
|
127
|
+
return sshUsername.contentEquals(username) && sshPassword.contentEquals(password);
|
|
119
128
|
}
|
|
120
129
|
});
|
|
121
130
|
sshServer.setPublickeyAuthenticator(new PublickeyAuthenticator()
|
|
@@ -133,10 +142,19 @@ public class TestSftpFileOutputPlugin
|
|
|
133
142
|
catch (IOException e) {
|
|
134
143
|
logger.debug(e.getMessage(), e);
|
|
135
144
|
}
|
|
145
|
+
return sshServer;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private HttpProxyServer createProxyServer(int port)
|
|
149
|
+
{
|
|
150
|
+
return DefaultHttpProxyServer.bootstrap()
|
|
151
|
+
.withPort(port)
|
|
152
|
+
.start();
|
|
136
153
|
}
|
|
137
154
|
|
|
138
155
|
@After
|
|
139
|
-
public void cleanup() throws InterruptedException
|
|
156
|
+
public void cleanup() throws InterruptedException
|
|
157
|
+
{
|
|
140
158
|
try {
|
|
141
159
|
sshServer.stop(true);
|
|
142
160
|
}
|
|
@@ -151,17 +169,19 @@ public class TestSftpFileOutputPlugin
|
|
|
151
169
|
return loader.fromYamlString(yaml);
|
|
152
170
|
}
|
|
153
171
|
|
|
154
|
-
private List<String> lsR(List<String> fileNames, Path dir)
|
|
172
|
+
private List<String> lsR(List<String> fileNames, Path dir)
|
|
173
|
+
{
|
|
155
174
|
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
|
|
156
175
|
for (Path path : stream) {
|
|
157
|
-
if(path.toFile().isDirectory()) {
|
|
176
|
+
if (path.toFile().isDirectory()) {
|
|
158
177
|
lsR(fileNames, path);
|
|
159
|
-
}
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
160
180
|
fileNames.add(path.toAbsolutePath().toString());
|
|
161
181
|
}
|
|
162
182
|
}
|
|
163
183
|
}
|
|
164
|
-
catch(IOException e) {
|
|
184
|
+
catch (IOException e) {
|
|
165
185
|
logger.debug(e.getMessage(), e);
|
|
166
186
|
}
|
|
167
187
|
return fileNames;
|
|
@@ -179,12 +199,12 @@ public class TestSftpFileOutputPlugin
|
|
|
179
199
|
boolean committed = false;
|
|
180
200
|
try {
|
|
181
201
|
// Result:
|
|
182
|
-
// _c0,_c1,_c2,_c3,_c4
|
|
183
|
-
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
|
184
|
-
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
|
185
|
-
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
|
186
|
-
|
|
187
|
-
|
|
202
|
+
// _c0,_c1,_c2,_c3,_c4,_c5
|
|
203
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
204
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
205
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
|
206
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
|
|
207
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
|
|
188
208
|
pageOutput.add(page);
|
|
189
209
|
if (sleep.isPresent()) {
|
|
190
210
|
Thread.sleep(sleep.get() * 1000);
|
|
@@ -215,7 +235,7 @@ public class TestSftpFileOutputPlugin
|
|
|
215
235
|
for (int i = 0; i < lines.size(); i++) {
|
|
216
236
|
String[] record = lines.get(i).split(",");
|
|
217
237
|
if (i == 0) {
|
|
218
|
-
for (int j = 0; j <= 4
|
|
238
|
+
for (int j = 0; j <= 4; j++) {
|
|
219
239
|
assertEquals("_c" + j, record[j]);
|
|
220
240
|
}
|
|
221
241
|
}
|
|
@@ -226,6 +246,7 @@ public class TestSftpFileOutputPlugin
|
|
|
226
246
|
assertEquals("3.0", record[2]);
|
|
227
247
|
assertEquals("45", record[3]);
|
|
228
248
|
assertEquals("1970-01-01 00:00:00.678000 +0000", record[4]);
|
|
249
|
+
assertEquals("{\"k\":\"v\"}", record[5]);
|
|
229
250
|
}
|
|
230
251
|
}
|
|
231
252
|
}
|
|
@@ -272,6 +293,49 @@ public class TestSftpFileOutputPlugin
|
|
|
272
293
|
assertEquals(pathPrefix, task.getPathPrefix());
|
|
273
294
|
assertEquals("txt", task.getFileNameExtension());
|
|
274
295
|
assertEquals("%03d.%02d.", task.getSequenceFormat());
|
|
296
|
+
assertEquals(Optional.absent(), task.getProxy());
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
@Test
|
|
300
|
+
public void testConfigValuesIncludingProxy()
|
|
301
|
+
{
|
|
302
|
+
// setting embulk config
|
|
303
|
+
final String pathPrefix = "/test/testUserPassword";
|
|
304
|
+
String configYaml = "" +
|
|
305
|
+
"type: sftp\n" +
|
|
306
|
+
"host: " + HOST + "\n" +
|
|
307
|
+
"user: " + USERNAME + "\n" +
|
|
308
|
+
"path_prefix: " + pathPrefix + "\n" +
|
|
309
|
+
"file_ext: txt\n" +
|
|
310
|
+
"proxy: \n" +
|
|
311
|
+
" type: http\n" +
|
|
312
|
+
" host: proxy_host\n" +
|
|
313
|
+
" port: 80 \n" +
|
|
314
|
+
" user: proxy_user\n" +
|
|
315
|
+
" password: proxy_pass\n" +
|
|
316
|
+
" command: proxy_command\n" +
|
|
317
|
+
"formatter:\n" +
|
|
318
|
+
" type: csv\n" +
|
|
319
|
+
" newline: CRLF\n" +
|
|
320
|
+
" newline_in_field: LF\n" +
|
|
321
|
+
" header_line: true\n" +
|
|
322
|
+
" charset: UTF-8\n" +
|
|
323
|
+
" quote_policy: NONE\n" +
|
|
324
|
+
" quote: \"\\\"\"\n" +
|
|
325
|
+
" escape: \"\\\\\"\n" +
|
|
326
|
+
" null_string: \"\"\n" +
|
|
327
|
+
" default_timezone: 'UTC'";
|
|
328
|
+
|
|
329
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
330
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
331
|
+
|
|
332
|
+
ProxyTask proxy = task.getProxy().get();
|
|
333
|
+
assertEquals("proxy_command", proxy.getCommand().get());
|
|
334
|
+
assertEquals("proxy_host", proxy.getHost().get());
|
|
335
|
+
assertEquals("proxy_user", proxy.getUser().get());
|
|
336
|
+
assertEquals("proxy_pass", proxy.getPassword().get());
|
|
337
|
+
assertEquals(80, proxy.getPort());
|
|
338
|
+
assertEquals(ProxyTask.ProxyType.HTTP, proxy.getType());
|
|
275
339
|
}
|
|
276
340
|
|
|
277
341
|
// Cases
|
|
@@ -284,7 +348,6 @@ public class TestSftpFileOutputPlugin
|
|
|
284
348
|
// timeout
|
|
285
349
|
// 0 second
|
|
286
350
|
|
|
287
|
-
|
|
288
351
|
@Test
|
|
289
352
|
public void testUserPasswordAndPutToUserDirectoryRoot()
|
|
290
353
|
{
|
|
@@ -318,7 +381,6 @@ public class TestSftpFileOutputPlugin
|
|
|
318
381
|
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
|
319
382
|
testFolder.getRoot().getAbsolutePath(),
|
|
320
383
|
pathPrefix));
|
|
321
|
-
|
|
322
384
|
}
|
|
323
385
|
|
|
324
386
|
@Test
|
|
@@ -358,6 +420,60 @@ public class TestSftpFileOutputPlugin
|
|
|
358
420
|
pathPrefix));
|
|
359
421
|
}
|
|
360
422
|
|
|
423
|
+
@Test
|
|
424
|
+
public void testUserSecretKeyFileWithProxy()
|
|
425
|
+
{
|
|
426
|
+
HttpProxyServer proxyServer = null;
|
|
427
|
+
try {
|
|
428
|
+
proxyServer = createProxyServer(PROXY_PORT);
|
|
429
|
+
|
|
430
|
+
// setting embulk config
|
|
431
|
+
final String pathPrefix = "/test/testUserPassword";
|
|
432
|
+
String configYaml = "" +
|
|
433
|
+
"type: sftp\n" +
|
|
434
|
+
"host: " + HOST + "\n" +
|
|
435
|
+
"port: " + PORT + "\n" +
|
|
436
|
+
"user: " + USERNAME + "\n" +
|
|
437
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
|
438
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
|
439
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
|
440
|
+
"file_ext: txt\n" +
|
|
441
|
+
"proxy: \n" +
|
|
442
|
+
" type: http\n" +
|
|
443
|
+
" host: " + PROXY_HOST + "\n" +
|
|
444
|
+
" port: " + PROXY_PORT + " \n" +
|
|
445
|
+
" user: " + USERNAME + "\n" +
|
|
446
|
+
" password: " + PASSWORD + "\n" +
|
|
447
|
+
" command: \n" +
|
|
448
|
+
"formatter:\n" +
|
|
449
|
+
" type: csv\n" +
|
|
450
|
+
" newline: CRLF\n" +
|
|
451
|
+
" newline_in_field: LF\n" +
|
|
452
|
+
" header_line: true\n" +
|
|
453
|
+
" charset: UTF-8\n" +
|
|
454
|
+
" quote_policy: NONE\n" +
|
|
455
|
+
" quote: \"\\\"\"\n" +
|
|
456
|
+
" escape: \"\\\\\"\n" +
|
|
457
|
+
" null_string: \"\"\n" +
|
|
458
|
+
" default_timezone: 'UTC'";
|
|
459
|
+
|
|
460
|
+
// runner.transaction -> ...
|
|
461
|
+
run(configYaml, Optional.<Integer>absent());
|
|
462
|
+
|
|
463
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
|
464
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
|
465
|
+
|
|
466
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
|
467
|
+
testFolder.getRoot().getAbsolutePath(),
|
|
468
|
+
pathPrefix));
|
|
469
|
+
}
|
|
470
|
+
finally {
|
|
471
|
+
if (proxyServer != null) {
|
|
472
|
+
proxyServer.stop();
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
361
477
|
@Test
|
|
362
478
|
public void testTimeout()
|
|
363
479
|
{
|
|
@@ -393,4 +509,30 @@ public class TestSftpFileOutputPlugin
|
|
|
393
509
|
// runner.transaction -> ...
|
|
394
510
|
run(configYaml, Optional.of(60)); // sleep 1 minute while processing
|
|
395
511
|
}
|
|
512
|
+
|
|
513
|
+
@Test
|
|
514
|
+
public void testProxyType()
|
|
515
|
+
{
|
|
516
|
+
// test valueOf()
|
|
517
|
+
assertEquals("http", ProxyTask.ProxyType.valueOf("HTTP").toString());
|
|
518
|
+
assertEquals("socks", ProxyTask.ProxyType.valueOf("SOCKS").toString());
|
|
519
|
+
assertEquals("stream", ProxyTask.ProxyType.valueOf("STREAM").toString());
|
|
520
|
+
try {
|
|
521
|
+
ProxyTask.ProxyType.valueOf("non-existing-type");
|
|
522
|
+
}
|
|
523
|
+
catch (Exception e) {
|
|
524
|
+
assertEquals(IllegalArgumentException.class, e.getClass());
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// test fromString
|
|
528
|
+
assertEquals(ProxyTask.ProxyType.HTTP, ProxyTask.ProxyType.fromString("http"));
|
|
529
|
+
assertEquals(ProxyTask.ProxyType.SOCKS, ProxyTask.ProxyType.fromString("socks"));
|
|
530
|
+
assertEquals(ProxyTask.ProxyType.STREAM, ProxyTask.ProxyType.fromString("stream"));
|
|
531
|
+
try {
|
|
532
|
+
ProxyTask.ProxyType.fromString("non-existing-type");
|
|
533
|
+
}
|
|
534
|
+
catch (Exception e) {
|
|
535
|
+
assertEquals(ConfigException.class, e.getClass());
|
|
536
|
+
}
|
|
537
|
+
}
|
|
396
538
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-sftp
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Civitaspo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-03-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -47,9 +47,12 @@ extra_rdoc_files: []
|
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
49
|
- .travis.yml
|
|
50
|
+
- CHANGELOG.md
|
|
50
51
|
- LICENSE.txt
|
|
51
52
|
- README.md
|
|
52
53
|
- build.gradle
|
|
54
|
+
- config/checkstyle/checkstyle.xml
|
|
55
|
+
- config/checkstyle/default.xml
|
|
53
56
|
- example/data.csv
|
|
54
57
|
- example/sample.yml
|
|
55
58
|
- gradle/wrapper/gradle-wrapper.jar
|
|
@@ -57,6 +60,7 @@ files:
|
|
|
57
60
|
- gradlew
|
|
58
61
|
- gradlew.bat
|
|
59
62
|
- lib/embulk/output/sftp.rb
|
|
63
|
+
- src/main/java/org/embulk/output/sftp/ProxyTask.java
|
|
60
64
|
- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
|
|
61
65
|
- src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
|
|
62
66
|
- src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
|
|
@@ -64,7 +68,7 @@ files:
|
|
|
64
68
|
- src/test/resources/id_rsa.pub
|
|
65
69
|
- classpath/commons-logging-1.2.jar
|
|
66
70
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
|
67
|
-
- classpath/embulk-output-sftp-0.0.
|
|
71
|
+
- classpath/embulk-output-sftp-0.0.5.jar
|
|
68
72
|
- classpath/jsch-0.1.53.jar
|
|
69
73
|
homepage: https://github.com/civitaspo/embulk-output-sftp
|
|
70
74
|
licenses:
|
|
Binary file
|