embulk-output-sftp 0.0.4 → 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +10 -0
- data/README.md +66 -2
- data/build.gradle +26 -6
- data/classpath/embulk-output-sftp-0.0.5.jar +0 -0
- data/config/checkstyle/checkstyle.xml +128 -0
- data/config/checkstyle/default.xml +108 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/sftp/ProxyTask.java +85 -0
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +62 -12
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +6 -1
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +173 -31
- metadata +7 -3
- data/classpath/embulk-output-sftp-0.0.4.jar +0 -0
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 865291280d7c6936d0e4f12655a00b854f880153
|
|
4
|
+
data.tar.gz: dfe086b9210d16e7da2e21d5f77e42c38bd7f23b
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: b226b3a68c6e81ddf17ff22577ffd80cae2b58201b18a38bae681b0729e531769426e3f760694d36121e1885a6dfc21383a91c4fe5664203aafb58ad8c00c0fd
|
|
7
|
+
data.tar.gz: 6bdda04f1ba31689656896323098be6e43e34c88cec3ea7922c6e0c55c6d8bc3b593f89ea863fcefb8a537da2ae38e6a2ebf159008e44e5d475cc025014fe80d
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
0.0.5 (2016-03-09)
|
|
2
|
+
==================
|
|
3
|
+
- Add: Support MapReduce executor
|
|
4
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/8
|
|
5
|
+
- Fix: Use ConfigException instead of RuntimeException
|
|
6
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/9/files
|
|
7
|
+
- Fix: Check to exist parent directory before uploading files
|
|
8
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/13
|
|
9
|
+
- Add: Support proxy settings
|
|
10
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/11
|
data/README.md
CHANGED
|
@@ -14,10 +14,10 @@ Stores files on a SFTP Server
|
|
|
14
14
|
## Configuration
|
|
15
15
|
|
|
16
16
|
- **host**: (string, required)
|
|
17
|
-
- **port**: (
|
|
17
|
+
- **port**: (int, default: `22`)
|
|
18
18
|
- **user**: (string, required)
|
|
19
19
|
- **password**: (string, default: `null`)
|
|
20
|
-
- **secret_key_file**: (string, default: `null`)
|
|
20
|
+
- **secret_key_file**: (string, default: `null`) [see below](#secret-keyfile-configuration)
|
|
21
21
|
- **secret_key_passphrase**: (string, default: `""`)
|
|
22
22
|
- **user_directory_is_root**: (boolean, default: `true`)
|
|
23
23
|
- **timeout**: sftp connection timeout seconds (integer, default: `600`)
|
|
@@ -25,6 +25,19 @@ Stores files on a SFTP Server
|
|
|
25
25
|
- **file_ext**: Extension of output files (string, required)
|
|
26
26
|
- **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
|
|
27
27
|
|
|
28
|
+
### Proxy configuration
|
|
29
|
+
|
|
30
|
+
- **proxy**:
|
|
31
|
+
- **type**: (string(http | socks | stream), required, default: `null`)
|
|
32
|
+
- **http**: use HTTP Proxy
|
|
33
|
+
- **socks**: use SOCKS Proxy
|
|
34
|
+
- **stream**: Connects to the SFTP server through a remote host reached by SSH
|
|
35
|
+
- **host**: (string, required)
|
|
36
|
+
- **port**: (int, default: `22`)
|
|
37
|
+
- **user**: (string, optional)
|
|
38
|
+
- **password**: (string, optional, default: `null`)
|
|
39
|
+
- **command**: (string, optional)
|
|
40
|
+
|
|
28
41
|
## Example
|
|
29
42
|
|
|
30
43
|
```yaml
|
|
@@ -42,6 +55,53 @@ out:
|
|
|
42
55
|
sequence_format: ".%01d%01d"
|
|
43
56
|
```
|
|
44
57
|
|
|
58
|
+
With proxy
|
|
59
|
+
```yaml
|
|
60
|
+
out:
|
|
61
|
+
type: sftp
|
|
62
|
+
host: 127.0.0.1
|
|
63
|
+
port: 22
|
|
64
|
+
user: embulk
|
|
65
|
+
secret_key_file: /Users/embulk/.ssh/id_rsa
|
|
66
|
+
secret_key_passphrase: secret_pass
|
|
67
|
+
user_directory_is_root: false
|
|
68
|
+
timeout: 600
|
|
69
|
+
path_prefix: /data/sftp
|
|
70
|
+
proxy:
|
|
71
|
+
type: http
|
|
72
|
+
host: proxy_host
|
|
73
|
+
port: 8080
|
|
74
|
+
user: proxy_user
|
|
75
|
+
password: proxy_secret_pass
|
|
76
|
+
command:
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
### Secret Keyfile configuration
|
|
80
|
+
|
|
81
|
+
Please set path of secret_key_file as follows.
|
|
82
|
+
```yaml
|
|
83
|
+
out:
|
|
84
|
+
type: sftp
|
|
85
|
+
...
|
|
86
|
+
secret_key_file: /path/to/id_rsa
|
|
87
|
+
...
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
You can also embed contents of secret_key_file at config.yml.
|
|
91
|
+
```yaml
|
|
92
|
+
out:
|
|
93
|
+
type: sftp
|
|
94
|
+
...
|
|
95
|
+
secret_key_file:
|
|
96
|
+
content |
|
|
97
|
+
-----BEGIN RSA PRIVATE KEY-----
|
|
98
|
+
ABCDEFG...
|
|
99
|
+
HIJKLMN...
|
|
100
|
+
OPQRSTU...
|
|
101
|
+
-----END RSA PRIVATE KEY-----
|
|
102
|
+
...
|
|
103
|
+
```
|
|
104
|
+
|
|
45
105
|
## Run Example
|
|
46
106
|
replace settings in `example/sample.yml` before running.
|
|
47
107
|
|
|
@@ -59,3 +119,7 @@ $ ./gradlew gem # -t to watch change of files and rebuild continuously
|
|
|
59
119
|
## Note
|
|
60
120
|
|
|
61
121
|
This plugin uses "org.apache.commons:commons-vfs" and the library uses the logger "org.apache.commons.logging.Log". So, this plugin suppress the logger's message except when embulk log level is debug.
|
|
122
|
+
|
|
123
|
+
## Contributors
|
|
124
|
+
- Satoshi Akama (@sakama)
|
|
125
|
+
- Rudolph Miller (@Rudolph-Miller)
|
data/build.gradle
CHANGED
|
@@ -4,6 +4,7 @@ plugins {
|
|
|
4
4
|
id "com.github.kt3k.coveralls" version "2.4.0"
|
|
5
5
|
id "jacoco"
|
|
6
6
|
id "java"
|
|
7
|
+
id "checkstyle"
|
|
7
8
|
}
|
|
8
9
|
import com.github.jrubygradle.JRubyExec
|
|
9
10
|
repositories {
|
|
@@ -14,20 +15,22 @@ configurations {
|
|
|
14
15
|
provided
|
|
15
16
|
}
|
|
16
17
|
|
|
17
|
-
version = "0.0.
|
|
18
|
+
version = "0.0.5"
|
|
18
19
|
sourceCompatibility = 1.7
|
|
19
20
|
targetCompatibility = 1.7
|
|
20
21
|
|
|
21
22
|
dependencies {
|
|
22
|
-
compile "org.embulk:embulk-core:0.
|
|
23
|
-
provided "org.embulk:embulk-core:0.
|
|
23
|
+
compile "org.embulk:embulk-core:0.8.6"
|
|
24
|
+
provided "org.embulk:embulk-core:0.8.6"
|
|
24
25
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
|
25
26
|
compile "org.apache.commons:commons-vfs2:2.+"
|
|
26
27
|
compile "com.jcraft:jsch:0.1.53"
|
|
27
28
|
testCompile "junit:junit:4.+"
|
|
28
|
-
testCompile "org.embulk:embulk-core:0.
|
|
29
|
-
testCompile "org.embulk:embulk-standards:0.
|
|
30
|
-
testCompile "org.apache.sshd:apache-sshd:1
|
|
29
|
+
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
|
30
|
+
testCompile "org.embulk:embulk-standards:0.8.6"
|
|
31
|
+
testCompile "org.apache.sshd:apache-sshd:1.1.0+"
|
|
32
|
+
testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
|
|
33
|
+
testCompile "io.netty:netty-all:4.0.34.Final"
|
|
31
34
|
}
|
|
32
35
|
|
|
33
36
|
jacocoTestReport {
|
|
@@ -44,6 +47,23 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
|
|
|
44
47
|
}
|
|
45
48
|
clean { delete "classpath" }
|
|
46
49
|
|
|
50
|
+
checkstyle {
|
|
51
|
+
configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
|
|
52
|
+
toolVersion = '6.14.1'
|
|
53
|
+
}
|
|
54
|
+
checkstyleMain {
|
|
55
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
56
|
+
ignoreFailures = true
|
|
57
|
+
}
|
|
58
|
+
checkstyleTest {
|
|
59
|
+
configFile = file("${project.rootDir}/config/checkstyle/default.xml")
|
|
60
|
+
ignoreFailures = true
|
|
61
|
+
}
|
|
62
|
+
task checkstyle(type: Checkstyle) {
|
|
63
|
+
classpath = sourceSets.main.output + sourceSets.test.output
|
|
64
|
+
source = sourceSets.main.allJava + sourceSets.test.allJava
|
|
65
|
+
}
|
|
66
|
+
|
|
47
67
|
task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
|
|
48
68
|
jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
|
|
49
69
|
script "${project.name}.gemspec"
|
|
Binary file
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<module name="Checker">
|
|
6
|
+
<!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
|
|
7
|
+
<module name="FileTabCharacter"/>
|
|
8
|
+
<module name="NewlineAtEndOfFile">
|
|
9
|
+
<property name="lineSeparator" value="lf"/>
|
|
10
|
+
</module>
|
|
11
|
+
<module name="RegexpMultiline">
|
|
12
|
+
<property name="format" value="\r"/>
|
|
13
|
+
<property name="message" value="Line contains carriage return"/>
|
|
14
|
+
</module>
|
|
15
|
+
<module name="RegexpMultiline">
|
|
16
|
+
<property name="format" value=" \n"/>
|
|
17
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
18
|
+
</module>
|
|
19
|
+
<module name="RegexpMultiline">
|
|
20
|
+
<property name="format" value="\{\n\n"/>
|
|
21
|
+
<property name="message" value="Blank line after opening brace"/>
|
|
22
|
+
</module>
|
|
23
|
+
<module name="RegexpMultiline">
|
|
24
|
+
<property name="format" value="\n\n\s*\}"/>
|
|
25
|
+
<property name="message" value="Blank line before closing brace"/>
|
|
26
|
+
</module>
|
|
27
|
+
<module name="RegexpMultiline">
|
|
28
|
+
<property name="format" value="\n\n\n"/>
|
|
29
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
30
|
+
</module>
|
|
31
|
+
<module name="RegexpMultiline">
|
|
32
|
+
<property name="format" value="\n\n\Z"/>
|
|
33
|
+
<property name="message" value="Blank line before end of file"/>
|
|
34
|
+
</module>
|
|
35
|
+
<module name="RegexpMultiline">
|
|
36
|
+
<property name="format" value="Preconditions\.checkNotNull"/>
|
|
37
|
+
<property name="message" value="Use of checkNotNull"/>
|
|
38
|
+
</module>
|
|
39
|
+
|
|
40
|
+
<module name="TreeWalker">
|
|
41
|
+
<module name="EmptyBlock">
|
|
42
|
+
<property name="option" value="text"/>
|
|
43
|
+
<property name="tokens" value="
|
|
44
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
45
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
46
|
+
</module>
|
|
47
|
+
<module name="EmptyStatement"/>
|
|
48
|
+
<module name="EmptyForInitializerPad"/>
|
|
49
|
+
<module name="EmptyForIteratorPad">
|
|
50
|
+
<property name="option" value="space"/>
|
|
51
|
+
</module>
|
|
52
|
+
<module name="MethodParamPad">
|
|
53
|
+
<property name="allowLineBreaks" value="true"/>
|
|
54
|
+
<property name="option" value="nospace"/>
|
|
55
|
+
</module>
|
|
56
|
+
<module name="ParenPad"/>
|
|
57
|
+
<module name="TypecastParenPad"/>
|
|
58
|
+
<module name="NeedBraces"/>
|
|
59
|
+
<module name="LeftCurly">
|
|
60
|
+
<property name="option" value="nl"/>
|
|
61
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
62
|
+
</module>
|
|
63
|
+
<module name="LeftCurly">
|
|
64
|
+
<property name="option" value="eol"/>
|
|
65
|
+
<property name="tokens" value="
|
|
66
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
67
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
68
|
+
</module>
|
|
69
|
+
<module name="RightCurly">
|
|
70
|
+
<property name="option" value="alone"/>
|
|
71
|
+
</module>
|
|
72
|
+
<module name="GenericWhitespace"/>
|
|
73
|
+
<module name="WhitespaceAfter"/>
|
|
74
|
+
<module name="NoWhitespaceBefore"/>
|
|
75
|
+
|
|
76
|
+
<module name="UpperEll"/>
|
|
77
|
+
<module name="DefaultComesLast"/>
|
|
78
|
+
<module name="ArrayTypeStyle"/>
|
|
79
|
+
<module name="MultipleVariableDeclarations"/>
|
|
80
|
+
<module name="ModifierOrder"/>
|
|
81
|
+
<module name="OneStatementPerLine"/>
|
|
82
|
+
<module name="StringLiteralEquality"/>
|
|
83
|
+
<module name="MutableException"/>
|
|
84
|
+
<module name="EqualsHashCode"/>
|
|
85
|
+
<module name="InnerAssignment"/>
|
|
86
|
+
<module name="InterfaceIsType"/>
|
|
87
|
+
<module name="HideUtilityClassConstructor"/>
|
|
88
|
+
|
|
89
|
+
<module name="MemberName"/>
|
|
90
|
+
<module name="LocalVariableName"/>
|
|
91
|
+
<module name="LocalFinalVariableName"/>
|
|
92
|
+
<module name="TypeName"/>
|
|
93
|
+
<module name="PackageName"/>
|
|
94
|
+
<module name="ParameterName"/>
|
|
95
|
+
<module name="StaticVariableName"/>
|
|
96
|
+
<module name="ClassTypeParameterName">
|
|
97
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
98
|
+
</module>
|
|
99
|
+
<module name="MethodTypeParameterName">
|
|
100
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
101
|
+
</module>
|
|
102
|
+
|
|
103
|
+
<module name="AvoidStarImport"/>
|
|
104
|
+
<module name="RedundantImport"/>
|
|
105
|
+
<module name="UnusedImports"/>
|
|
106
|
+
<module name="ImportOrder">
|
|
107
|
+
<property name="groups" value="*,javax,java"/>
|
|
108
|
+
<property name="separated" value="true"/>
|
|
109
|
+
<property name="option" value="bottom"/>
|
|
110
|
+
<property name="sortStaticImportsAlphabetically" value="true"/>
|
|
111
|
+
</module>
|
|
112
|
+
|
|
113
|
+
<module name="WhitespaceAround">
|
|
114
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
115
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
116
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
117
|
+
<property name="tokens" value="
|
|
118
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
119
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
120
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
121
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
122
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
123
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
124
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
125
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
126
|
+
</module>
|
|
127
|
+
</module>
|
|
128
|
+
</module>
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
|
2
|
+
<!DOCTYPE module PUBLIC
|
|
3
|
+
"-//Puppy Crawl//DTD Check Configuration 1.3//EN"
|
|
4
|
+
"http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
|
|
5
|
+
<!--
|
|
6
|
+
This is a subset of ./checkstyle.xml which allows some loose styles
|
|
7
|
+
-->
|
|
8
|
+
<module name="Checker">
|
|
9
|
+
<module name="FileTabCharacter"/>
|
|
10
|
+
<module name="NewlineAtEndOfFile">
|
|
11
|
+
<property name="lineSeparator" value="lf"/>
|
|
12
|
+
</module>
|
|
13
|
+
<module name="RegexpMultiline">
|
|
14
|
+
<property name="format" value="\r"/>
|
|
15
|
+
<property name="message" value="Line contains carriage return"/>
|
|
16
|
+
</module>
|
|
17
|
+
<module name="RegexpMultiline">
|
|
18
|
+
<property name="format" value=" \n"/>
|
|
19
|
+
<property name="message" value="Line has trailing whitespace"/>
|
|
20
|
+
</module>
|
|
21
|
+
<module name="RegexpMultiline">
|
|
22
|
+
<property name="format" value="\n\n\n"/>
|
|
23
|
+
<property name="message" value="Multiple consecutive blank lines"/>
|
|
24
|
+
</module>
|
|
25
|
+
<module name="RegexpMultiline">
|
|
26
|
+
<property name="format" value="\n\n\Z"/>
|
|
27
|
+
<property name="message" value="Blank line before end of file"/>
|
|
28
|
+
</module>
|
|
29
|
+
|
|
30
|
+
<module name="TreeWalker">
|
|
31
|
+
<module name="EmptyBlock">
|
|
32
|
+
<property name="option" value="text"/>
|
|
33
|
+
<property name="tokens" value="
|
|
34
|
+
LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
|
|
35
|
+
LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
|
|
36
|
+
</module>
|
|
37
|
+
<module name="EmptyStatement"/>
|
|
38
|
+
<module name="EmptyForInitializerPad"/>
|
|
39
|
+
<module name="EmptyForIteratorPad">
|
|
40
|
+
<property name="option" value="space"/>
|
|
41
|
+
</module>
|
|
42
|
+
<module name="MethodParamPad">
|
|
43
|
+
<property name="allowLineBreaks" value="true"/>
|
|
44
|
+
<property name="option" value="nospace"/>
|
|
45
|
+
</module>
|
|
46
|
+
<module name="ParenPad"/>
|
|
47
|
+
<module name="TypecastParenPad"/>
|
|
48
|
+
<module name="NeedBraces"/>
|
|
49
|
+
<module name="LeftCurly">
|
|
50
|
+
<property name="option" value="nl"/>
|
|
51
|
+
<property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
|
|
52
|
+
</module>
|
|
53
|
+
<module name="LeftCurly">
|
|
54
|
+
<property name="option" value="eol"/>
|
|
55
|
+
<property name="tokens" value="
|
|
56
|
+
LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
|
|
57
|
+
LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
|
|
58
|
+
</module>
|
|
59
|
+
<module name="RightCurly">
|
|
60
|
+
<property name="option" value="alone"/>
|
|
61
|
+
</module>
|
|
62
|
+
<module name="GenericWhitespace"/>
|
|
63
|
+
<module name="WhitespaceAfter"/>
|
|
64
|
+
<module name="NoWhitespaceBefore"/>
|
|
65
|
+
|
|
66
|
+
<module name="UpperEll"/>
|
|
67
|
+
<module name="DefaultComesLast"/>
|
|
68
|
+
<module name="ArrayTypeStyle"/>
|
|
69
|
+
<module name="MultipleVariableDeclarations"/>
|
|
70
|
+
<module name="ModifierOrder"/>
|
|
71
|
+
<module name="OneStatementPerLine"/>
|
|
72
|
+
<module name="StringLiteralEquality"/>
|
|
73
|
+
<module name="MutableException"/>
|
|
74
|
+
<module name="EqualsHashCode"/>
|
|
75
|
+
<module name="InnerAssignment"/>
|
|
76
|
+
<module name="InterfaceIsType"/>
|
|
77
|
+
<module name="HideUtilityClassConstructor"/>
|
|
78
|
+
|
|
79
|
+
<module name="MemberName"/>
|
|
80
|
+
<module name="LocalVariableName"/>
|
|
81
|
+
<module name="LocalFinalVariableName"/>
|
|
82
|
+
<module name="TypeName"/>
|
|
83
|
+
<module name="PackageName"/>
|
|
84
|
+
<module name="ParameterName"/>
|
|
85
|
+
<module name="StaticVariableName"/>
|
|
86
|
+
<module name="ClassTypeParameterName">
|
|
87
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
88
|
+
</module>
|
|
89
|
+
<module name="MethodTypeParameterName">
|
|
90
|
+
<property name="format" value="^[A-Z][0-9]?$"/>
|
|
91
|
+
</module>
|
|
92
|
+
|
|
93
|
+
<module name="WhitespaceAround">
|
|
94
|
+
<property name="allowEmptyConstructors" value="true"/>
|
|
95
|
+
<property name="allowEmptyMethods" value="true"/>
|
|
96
|
+
<property name="ignoreEnhancedForColon" value="false"/>
|
|
97
|
+
<property name="tokens" value="
|
|
98
|
+
ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
|
|
99
|
+
BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
|
|
100
|
+
LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
|
|
101
|
+
LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
|
|
102
|
+
LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
|
|
103
|
+
LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
|
|
104
|
+
PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
|
|
105
|
+
STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
|
|
106
|
+
</module>
|
|
107
|
+
</module>
|
|
108
|
+
</module>
|
|
Binary file
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
#
|
|
1
|
+
#Wed Jan 13 12:41:02 JST 2016
|
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
|
3
3
|
distributionPath=wrapper/dists
|
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
|
5
5
|
zipStorePath=wrapper/dists
|
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
package org.embulk.output.sftp;
|
|
2
|
+
|
|
3
|
+
import com.fasterxml.jackson.annotation.JsonCreator;
|
|
4
|
+
import com.fasterxml.jackson.annotation.JsonValue;
|
|
5
|
+
import com.google.common.base.Optional;
|
|
6
|
+
import org.apache.commons.vfs2.FileSystemOptions;
|
|
7
|
+
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
|
8
|
+
import org.embulk.config.Config;
|
|
9
|
+
import org.embulk.config.ConfigDefault;
|
|
10
|
+
import org.embulk.config.ConfigException;
|
|
11
|
+
import org.embulk.config.Task;
|
|
12
|
+
|
|
13
|
+
import java.util.Locale;
|
|
14
|
+
|
|
15
|
+
interface ProxyTask
|
|
16
|
+
extends Task
|
|
17
|
+
{
|
|
18
|
+
@Config("type")
|
|
19
|
+
public ProxyType getType();
|
|
20
|
+
|
|
21
|
+
@Config("host")
|
|
22
|
+
public Optional<String> getHost();
|
|
23
|
+
|
|
24
|
+
@Config("user")
|
|
25
|
+
@ConfigDefault("null")
|
|
26
|
+
public Optional<String> getUser();
|
|
27
|
+
|
|
28
|
+
@Config("password")
|
|
29
|
+
@ConfigDefault("null")
|
|
30
|
+
public Optional<String> getPassword();
|
|
31
|
+
|
|
32
|
+
@Config("port")
|
|
33
|
+
@ConfigDefault("22")
|
|
34
|
+
public int getPort();
|
|
35
|
+
|
|
36
|
+
@Config("command")
|
|
37
|
+
@ConfigDefault("null")
|
|
38
|
+
public Optional<String> getCommand();
|
|
39
|
+
|
|
40
|
+
public enum ProxyType
|
|
41
|
+
{
|
|
42
|
+
HTTP,
|
|
43
|
+
SOCKS,
|
|
44
|
+
STREAM;
|
|
45
|
+
|
|
46
|
+
@JsonValue
|
|
47
|
+
@Override
|
|
48
|
+
public String toString()
|
|
49
|
+
{
|
|
50
|
+
return name().toLowerCase(Locale.ENGLISH);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
@JsonCreator
|
|
54
|
+
public static ProxyType fromString(String value)
|
|
55
|
+
{
|
|
56
|
+
switch (value) {
|
|
57
|
+
case "http":
|
|
58
|
+
return HTTP;
|
|
59
|
+
case "socks":
|
|
60
|
+
return SOCKS;
|
|
61
|
+
case "stream":
|
|
62
|
+
return STREAM;
|
|
63
|
+
default:
|
|
64
|
+
throw new ConfigException(String.format("Unknown proxy type '%s'. Supported proxy types are http, socks, stream", value));
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public static SftpFileSystemConfigBuilder setProxyType(SftpFileSystemConfigBuilder builder, FileSystemOptions fsOptions, ProxyTask.ProxyType type)
|
|
69
|
+
{
|
|
70
|
+
SftpFileSystemConfigBuilder.ProxyType setType = null;
|
|
71
|
+
switch (type) {
|
|
72
|
+
case HTTP:
|
|
73
|
+
setType = SftpFileSystemConfigBuilder.PROXY_HTTP;
|
|
74
|
+
break;
|
|
75
|
+
case SOCKS:
|
|
76
|
+
setType = SftpFileSystemConfigBuilder.PROXY_SOCKS5;
|
|
77
|
+
break;
|
|
78
|
+
case STREAM:
|
|
79
|
+
setType = SftpFileSystemConfigBuilder.PROXY_STREAM;
|
|
80
|
+
}
|
|
81
|
+
builder.setProxyType(fsOptions, setType);
|
|
82
|
+
return builder;
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
package org.embulk.output.sftp;
|
|
2
2
|
|
|
3
|
+
import com.google.common.base.Function;
|
|
3
4
|
import com.google.common.base.Throwables;
|
|
4
5
|
import org.apache.commons.vfs2.FileObject;
|
|
5
6
|
import org.apache.commons.vfs2.FileSystemException;
|
|
@@ -7,11 +8,13 @@ import org.apache.commons.vfs2.FileSystemOptions;
|
|
|
7
8
|
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
|
8
9
|
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
|
9
10
|
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
|
11
|
+
import org.embulk.config.ConfigException;
|
|
10
12
|
import org.embulk.config.TaskReport;
|
|
11
13
|
import org.embulk.spi.Buffer;
|
|
12
14
|
import org.embulk.spi.Exec;
|
|
13
15
|
import org.embulk.spi.FileOutput;
|
|
14
16
|
import org.embulk.spi.TransactionalFileOutput;
|
|
17
|
+
import org.embulk.spi.unit.LocalFile;
|
|
15
18
|
import org.slf4j.Logger;
|
|
16
19
|
|
|
17
20
|
import java.io.File;
|
|
@@ -56,7 +59,7 @@ public class SftpFileOutput
|
|
|
56
59
|
}
|
|
57
60
|
catch (FileSystemException e) {
|
|
58
61
|
logger.error(e.getMessage());
|
|
59
|
-
throw new
|
|
62
|
+
throw new ConfigException(e);
|
|
60
63
|
}
|
|
61
64
|
|
|
62
65
|
return manager;
|
|
@@ -76,18 +79,45 @@ public class SftpFileOutput
|
|
|
76
79
|
FileSystemOptions fsOptions = new FileSystemOptions();
|
|
77
80
|
|
|
78
81
|
try {
|
|
79
|
-
SftpFileSystemConfigBuilder.getInstance()
|
|
80
|
-
|
|
81
|
-
|
|
82
|
+
SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
|
|
83
|
+
builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
|
|
84
|
+
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout());
|
|
85
|
+
builder.setStrictHostKeyChecking(fsOptions, "no");
|
|
82
86
|
if (task.getSecretKeyFilePath().isPresent()) {
|
|
83
|
-
IdentityInfo identityInfo = new IdentityInfo(
|
|
84
|
-
|
|
87
|
+
IdentityInfo identityInfo = new IdentityInfo(
|
|
88
|
+
new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
|
|
89
|
+
task.getSecretKeyPassphrase().getBytes()
|
|
90
|
+
);
|
|
91
|
+
builder.setIdentityInfo(fsOptions, identityInfo);
|
|
85
92
|
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
|
86
93
|
}
|
|
94
|
+
|
|
95
|
+
if (task.getProxy().isPresent()) {
|
|
96
|
+
ProxyTask proxy = task.getProxy().get();
|
|
97
|
+
|
|
98
|
+
ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
|
|
99
|
+
|
|
100
|
+
if (proxy.getHost().isPresent()) {
|
|
101
|
+
builder.setProxyHost(fsOptions, proxy.getHost().get());
|
|
102
|
+
builder.setProxyPort(fsOptions, proxy.getPort());
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (proxy.getUser().isPresent()) {
|
|
106
|
+
builder.setProxyUser(fsOptions, proxy.getUser().get());
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (proxy.getPassword().isPresent()) {
|
|
110
|
+
builder.setProxyPassword(fsOptions, proxy.getPassword().get());
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
if (proxy.getCommand().isPresent()) {
|
|
114
|
+
builder.setProxyCommand(fsOptions, proxy.getCommand().get());
|
|
115
|
+
}
|
|
116
|
+
}
|
|
87
117
|
}
|
|
88
118
|
catch (FileSystemException e) {
|
|
89
119
|
logger.error(e.getMessage());
|
|
90
|
-
throw new
|
|
120
|
+
throw new ConfigException(e);
|
|
91
121
|
}
|
|
92
122
|
|
|
93
123
|
return fsOptions;
|
|
@@ -137,7 +167,9 @@ public class SftpFileOutput
|
|
|
137
167
|
logger.error(e.getMessage());
|
|
138
168
|
Throwables.propagate(e);
|
|
139
169
|
}
|
|
140
|
-
|
|
170
|
+
finally {
|
|
171
|
+
buffer.release();
|
|
172
|
+
}
|
|
141
173
|
}
|
|
142
174
|
|
|
143
175
|
@Override
|
|
@@ -161,10 +193,9 @@ public class SftpFileOutput
|
|
|
161
193
|
@Override
|
|
162
194
|
public TaskReport commit()
|
|
163
195
|
{
|
|
164
|
-
return
|
|
196
|
+
return Exec.newTaskReport();
|
|
165
197
|
}
|
|
166
198
|
|
|
167
|
-
|
|
168
199
|
private void closeCurrentFile()
|
|
169
200
|
{
|
|
170
201
|
if (currentFile == null) {
|
|
@@ -194,7 +225,7 @@ public class SftpFileOutput
|
|
|
194
225
|
}
|
|
195
226
|
catch (URISyntaxException e) {
|
|
196
227
|
logger.error(e.getMessage());
|
|
197
|
-
throw new
|
|
228
|
+
throw new ConfigException(e);
|
|
198
229
|
}
|
|
199
230
|
}
|
|
200
231
|
|
|
@@ -209,7 +240,15 @@ public class SftpFileOutput
|
|
|
209
240
|
int count = 0;
|
|
210
241
|
while (true) {
|
|
211
242
|
try {
|
|
212
|
-
|
|
243
|
+
FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
|
|
244
|
+
if (file.getParent().exists()) {
|
|
245
|
+
logger.info("parent directory {} exists there", file.getParent());
|
|
246
|
+
return file;
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
logger.info("trying to create parent directory {}", file.getParent());
|
|
250
|
+
file.getParent().createFolder();
|
|
251
|
+
}
|
|
213
252
|
}
|
|
214
253
|
catch (FileSystemException e) {
|
|
215
254
|
if (++count == maxConnectionRetry) {
|
|
@@ -230,4 +269,15 @@ public class SftpFileOutput
|
|
|
230
269
|
}
|
|
231
270
|
}
|
|
232
271
|
}
|
|
272
|
+
|
|
273
|
+
private Function<LocalFile, String> localFileToPathString()
|
|
274
|
+
{
|
|
275
|
+
return new Function<LocalFile, String>()
|
|
276
|
+
{
|
|
277
|
+
public String apply(LocalFile file)
|
|
278
|
+
{
|
|
279
|
+
return file.getPath().toString();
|
|
280
|
+
}
|
|
281
|
+
};
|
|
282
|
+
}
|
|
233
283
|
}
|
|
@@ -11,6 +11,7 @@ import org.embulk.config.TaskSource;
|
|
|
11
11
|
import org.embulk.spi.Exec;
|
|
12
12
|
import org.embulk.spi.FileOutputPlugin;
|
|
13
13
|
import org.embulk.spi.TransactionalFileOutput;
|
|
14
|
+
import org.embulk.spi.unit.LocalFile;
|
|
14
15
|
import org.slf4j.Logger;
|
|
15
16
|
|
|
16
17
|
import java.util.List;
|
|
@@ -39,7 +40,8 @@ public class SftpFileOutputPlugin
|
|
|
39
40
|
|
|
40
41
|
@Config("secret_key_file")
|
|
41
42
|
@ConfigDefault("null")
|
|
42
|
-
public Optional<
|
|
43
|
+
public Optional<LocalFile> getSecretKeyFilePath();
|
|
44
|
+
public void setSecretKeyFilePath(Optional<LocalFile> secretKeyFilePath);
|
|
43
45
|
|
|
44
46
|
@Config("secret_key_passphrase")
|
|
45
47
|
@ConfigDefault("\"\"")
|
|
@@ -67,6 +69,9 @@ public class SftpFileOutputPlugin
|
|
|
67
69
|
@ConfigDefault("\"%03d.%02d.\"")
|
|
68
70
|
public String getSequenceFormat();
|
|
69
71
|
|
|
72
|
+
@Config("proxy")
|
|
73
|
+
@ConfigDefault("null")
|
|
74
|
+
public Optional<ProxyTask> getProxy();
|
|
70
75
|
}
|
|
71
76
|
|
|
72
77
|
@Override
|
|
@@ -2,8 +2,6 @@ package org.embulk.output.sftp;
|
|
|
2
2
|
|
|
3
3
|
import com.google.common.base.Charsets;
|
|
4
4
|
import com.google.common.base.Optional;
|
|
5
|
-
import com.google.common.base.Splitter;
|
|
6
|
-
import com.google.common.base.Throwables;
|
|
7
5
|
import com.google.common.collect.Lists;
|
|
8
6
|
import com.google.common.io.Resources;
|
|
9
7
|
import org.apache.commons.vfs2.FileSystemException;
|
|
@@ -13,11 +11,12 @@ import org.apache.sshd.server.Command;
|
|
|
13
11
|
import org.apache.sshd.server.SshServer;
|
|
14
12
|
import org.apache.sshd.server.auth.password.PasswordAuthenticator;
|
|
15
13
|
import org.apache.sshd.server.auth.pubkey.PublickeyAuthenticator;
|
|
16
|
-
import org.apache.sshd.server.command.ScpCommandFactory;
|
|
17
14
|
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
|
|
15
|
+
import org.apache.sshd.server.scp.ScpCommandFactory;
|
|
18
16
|
import org.apache.sshd.server.session.ServerSession;
|
|
19
17
|
import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory;
|
|
20
18
|
import org.embulk.EmbulkTestRuntime;
|
|
19
|
+
import org.embulk.config.ConfigException;
|
|
21
20
|
import org.embulk.config.ConfigLoader;
|
|
22
21
|
import org.embulk.config.ConfigSource;
|
|
23
22
|
import org.embulk.config.TaskReport;
|
|
@@ -32,37 +31,39 @@ import org.embulk.spi.Schema;
|
|
|
32
31
|
import org.embulk.spi.TransactionalPageOutput;
|
|
33
32
|
import org.embulk.spi.time.Timestamp;
|
|
34
33
|
import org.hamcrest.CoreMatchers;
|
|
35
|
-
import org.hamcrest.Matcher;
|
|
36
34
|
import org.junit.After;
|
|
37
35
|
import org.junit.Before;
|
|
38
36
|
import org.junit.Rule;
|
|
39
37
|
import org.junit.Test;
|
|
40
38
|
import org.junit.rules.ExpectedException;
|
|
41
39
|
import org.junit.rules.TemporaryFolder;
|
|
40
|
+
import org.littleshoot.proxy.HttpProxyServer;
|
|
41
|
+
import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
|
|
42
42
|
import org.slf4j.Logger;
|
|
43
43
|
|
|
44
44
|
import java.io.File;
|
|
45
45
|
import java.io.IOException;
|
|
46
46
|
import java.nio.file.DirectoryStream;
|
|
47
|
-
import java.nio.file.FileSystems;
|
|
48
47
|
import java.nio.file.Files;
|
|
49
48
|
import java.nio.file.Path;
|
|
50
|
-
import java.nio.file.PathMatcher;
|
|
51
49
|
import java.nio.file.Paths;
|
|
52
50
|
import java.security.PublicKey;
|
|
53
|
-
import java.util.Arrays;
|
|
54
51
|
import java.util.Collections;
|
|
55
52
|
import java.util.List;
|
|
56
53
|
|
|
57
54
|
import static com.google.common.io.Files.readLines;
|
|
58
|
-
import static org.embulk.spi.type.Types
|
|
55
|
+
import static org.embulk.spi.type.Types.BOOLEAN;
|
|
56
|
+
import static org.embulk.spi.type.Types.DOUBLE;
|
|
57
|
+
import static org.embulk.spi.type.Types.JSON;
|
|
58
|
+
import static org.embulk.spi.type.Types.LONG;
|
|
59
|
+
import static org.embulk.spi.type.Types.STRING;
|
|
60
|
+
import static org.embulk.spi.type.Types.TIMESTAMP;
|
|
59
61
|
import static org.hamcrest.CoreMatchers.containsString;
|
|
60
62
|
import static org.hamcrest.CoreMatchers.hasItem;
|
|
61
|
-
import static org.hamcrest.CoreMatchers.instanceOf;
|
|
62
|
-
import static org.hamcrest.MatcherAssert.*;
|
|
63
|
-
import static org.hamcrest.core.Is.is;
|
|
64
63
|
import static org.junit.Assert.assertEquals;
|
|
65
|
-
|
|
64
|
+
import static org.junit.Assert.assertThat;
|
|
65
|
+
import static org.msgpack.value.ValueFactory.newMap;
|
|
66
|
+
import static org.msgpack.value.ValueFactory.newString;
|
|
66
67
|
|
|
67
68
|
public class TestSftpFileOutputPlugin
|
|
68
69
|
{
|
|
@@ -80,6 +81,8 @@ public class TestSftpFileOutputPlugin
|
|
|
80
81
|
private SshServer sshServer;
|
|
81
82
|
private static final String HOST = "127.0.0.1";
|
|
82
83
|
private static final int PORT = 20022;
|
|
84
|
+
private static final String PROXY_HOST = "127.0.0.1";
|
|
85
|
+
private static final int PROXY_PORT = 8080;
|
|
83
86
|
private static final String USERNAME = "username";
|
|
84
87
|
private static final String PASSWORD = "password";
|
|
85
88
|
private static final String SECRET_KEY_FILE = Resources.getResource("id_rsa").getPath();
|
|
@@ -90,6 +93,7 @@ public class TestSftpFileOutputPlugin
|
|
|
90
93
|
.add("_c2", DOUBLE)
|
|
91
94
|
.add("_c3", STRING)
|
|
92
95
|
.add("_c4", TIMESTAMP)
|
|
96
|
+
.add("_c5", JSON)
|
|
93
97
|
.build();
|
|
94
98
|
|
|
95
99
|
@Before
|
|
@@ -100,13 +104,18 @@ public class TestSftpFileOutputPlugin
|
|
|
100
104
|
SftpFileOutputPlugin sftpFileOutputPlugin = new SftpFileOutputPlugin();
|
|
101
105
|
runner = new FileOutputRunner(sftpFileOutputPlugin);
|
|
102
106
|
|
|
107
|
+
sshServer = createSshServer(HOST, PORT, USERNAME, PASSWORD);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
private SshServer createSshServer(String host, int port, final String sshUsername, final String sshPassword)
|
|
111
|
+
{
|
|
103
112
|
// setup a mock sftp server
|
|
104
|
-
sshServer = SshServer.setUpDefaultServer();
|
|
113
|
+
SshServer sshServer = SshServer.setUpDefaultServer();
|
|
105
114
|
VirtualFileSystemFactory fsFactory = new VirtualFileSystemFactory();
|
|
106
|
-
fsFactory.setUserHomeDir(
|
|
115
|
+
fsFactory.setUserHomeDir(sshUsername, testFolder.getRoot().toPath());
|
|
107
116
|
sshServer.setFileSystemFactory(fsFactory);
|
|
108
|
-
sshServer.setHost(
|
|
109
|
-
sshServer.setPort(
|
|
117
|
+
sshServer.setHost(host);
|
|
118
|
+
sshServer.setPort(port);
|
|
110
119
|
sshServer.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystemFactory()));
|
|
111
120
|
sshServer.setCommandFactory(new ScpCommandFactory());
|
|
112
121
|
sshServer.setKeyPairProvider(new SimpleGeneratorHostKeyProvider());
|
|
@@ -115,7 +124,7 @@ public class TestSftpFileOutputPlugin
|
|
|
115
124
|
@Override
|
|
116
125
|
public boolean authenticate(final String username, final String password, final ServerSession session)
|
|
117
126
|
{
|
|
118
|
-
return
|
|
127
|
+
return sshUsername.contentEquals(username) && sshPassword.contentEquals(password);
|
|
119
128
|
}
|
|
120
129
|
});
|
|
121
130
|
sshServer.setPublickeyAuthenticator(new PublickeyAuthenticator()
|
|
@@ -133,10 +142,19 @@ public class TestSftpFileOutputPlugin
|
|
|
133
142
|
catch (IOException e) {
|
|
134
143
|
logger.debug(e.getMessage(), e);
|
|
135
144
|
}
|
|
145
|
+
return sshServer;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private HttpProxyServer createProxyServer(int port)
|
|
149
|
+
{
|
|
150
|
+
return DefaultHttpProxyServer.bootstrap()
|
|
151
|
+
.withPort(port)
|
|
152
|
+
.start();
|
|
136
153
|
}
|
|
137
154
|
|
|
138
155
|
@After
|
|
139
|
-
public void cleanup() throws InterruptedException
|
|
156
|
+
public void cleanup() throws InterruptedException
|
|
157
|
+
{
|
|
140
158
|
try {
|
|
141
159
|
sshServer.stop(true);
|
|
142
160
|
}
|
|
@@ -151,17 +169,19 @@ public class TestSftpFileOutputPlugin
|
|
|
151
169
|
return loader.fromYamlString(yaml);
|
|
152
170
|
}
|
|
153
171
|
|
|
154
|
-
private List<String> lsR(List<String> fileNames, Path dir)
|
|
172
|
+
private List<String> lsR(List<String> fileNames, Path dir)
|
|
173
|
+
{
|
|
155
174
|
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
|
|
156
175
|
for (Path path : stream) {
|
|
157
|
-
if(path.toFile().isDirectory()) {
|
|
176
|
+
if (path.toFile().isDirectory()) {
|
|
158
177
|
lsR(fileNames, path);
|
|
159
|
-
}
|
|
178
|
+
}
|
|
179
|
+
else {
|
|
160
180
|
fileNames.add(path.toAbsolutePath().toString());
|
|
161
181
|
}
|
|
162
182
|
}
|
|
163
183
|
}
|
|
164
|
-
catch(IOException e) {
|
|
184
|
+
catch (IOException e) {
|
|
165
185
|
logger.debug(e.getMessage(), e);
|
|
166
186
|
}
|
|
167
187
|
return fileNames;
|
|
@@ -179,12 +199,12 @@ public class TestSftpFileOutputPlugin
|
|
|
179
199
|
boolean committed = false;
|
|
180
200
|
try {
|
|
181
201
|
// Result:
|
|
182
|
-
// _c0,_c1,_c2,_c3,_c4
|
|
183
|
-
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
|
184
|
-
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
|
185
|
-
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
|
186
|
-
|
|
187
|
-
|
|
202
|
+
// _c0,_c1,_c2,_c3,_c4,_c5
|
|
203
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
204
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
|
205
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
|
206
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
|
|
207
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
|
|
188
208
|
pageOutput.add(page);
|
|
189
209
|
if (sleep.isPresent()) {
|
|
190
210
|
Thread.sleep(sleep.get() * 1000);
|
|
@@ -215,7 +235,7 @@ public class TestSftpFileOutputPlugin
|
|
|
215
235
|
for (int i = 0; i < lines.size(); i++) {
|
|
216
236
|
String[] record = lines.get(i).split(",");
|
|
217
237
|
if (i == 0) {
|
|
218
|
-
for (int j = 0; j <= 4
|
|
238
|
+
for (int j = 0; j <= 4; j++) {
|
|
219
239
|
assertEquals("_c" + j, record[j]);
|
|
220
240
|
}
|
|
221
241
|
}
|
|
@@ -226,6 +246,7 @@ public class TestSftpFileOutputPlugin
|
|
|
226
246
|
assertEquals("3.0", record[2]);
|
|
227
247
|
assertEquals("45", record[3]);
|
|
228
248
|
assertEquals("1970-01-01 00:00:00.678000 +0000", record[4]);
|
|
249
|
+
assertEquals("{\"k\":\"v\"}", record[5]);
|
|
229
250
|
}
|
|
230
251
|
}
|
|
231
252
|
}
|
|
@@ -272,6 +293,49 @@ public class TestSftpFileOutputPlugin
|
|
|
272
293
|
assertEquals(pathPrefix, task.getPathPrefix());
|
|
273
294
|
assertEquals("txt", task.getFileNameExtension());
|
|
274
295
|
assertEquals("%03d.%02d.", task.getSequenceFormat());
|
|
296
|
+
assertEquals(Optional.absent(), task.getProxy());
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
@Test
|
|
300
|
+
public void testConfigValuesIncludingProxy()
|
|
301
|
+
{
|
|
302
|
+
// setting embulk config
|
|
303
|
+
final String pathPrefix = "/test/testUserPassword";
|
|
304
|
+
String configYaml = "" +
|
|
305
|
+
"type: sftp\n" +
|
|
306
|
+
"host: " + HOST + "\n" +
|
|
307
|
+
"user: " + USERNAME + "\n" +
|
|
308
|
+
"path_prefix: " + pathPrefix + "\n" +
|
|
309
|
+
"file_ext: txt\n" +
|
|
310
|
+
"proxy: \n" +
|
|
311
|
+
" type: http\n" +
|
|
312
|
+
" host: proxy_host\n" +
|
|
313
|
+
" port: 80 \n" +
|
|
314
|
+
" user: proxy_user\n" +
|
|
315
|
+
" password: proxy_pass\n" +
|
|
316
|
+
" command: proxy_command\n" +
|
|
317
|
+
"formatter:\n" +
|
|
318
|
+
" type: csv\n" +
|
|
319
|
+
" newline: CRLF\n" +
|
|
320
|
+
" newline_in_field: LF\n" +
|
|
321
|
+
" header_line: true\n" +
|
|
322
|
+
" charset: UTF-8\n" +
|
|
323
|
+
" quote_policy: NONE\n" +
|
|
324
|
+
" quote: \"\\\"\"\n" +
|
|
325
|
+
" escape: \"\\\\\"\n" +
|
|
326
|
+
" null_string: \"\"\n" +
|
|
327
|
+
" default_timezone: 'UTC'";
|
|
328
|
+
|
|
329
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
|
330
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
|
331
|
+
|
|
332
|
+
ProxyTask proxy = task.getProxy().get();
|
|
333
|
+
assertEquals("proxy_command", proxy.getCommand().get());
|
|
334
|
+
assertEquals("proxy_host", proxy.getHost().get());
|
|
335
|
+
assertEquals("proxy_user", proxy.getUser().get());
|
|
336
|
+
assertEquals("proxy_pass", proxy.getPassword().get());
|
|
337
|
+
assertEquals(80, proxy.getPort());
|
|
338
|
+
assertEquals(ProxyTask.ProxyType.HTTP, proxy.getType());
|
|
275
339
|
}
|
|
276
340
|
|
|
277
341
|
// Cases
|
|
@@ -284,7 +348,6 @@ public class TestSftpFileOutputPlugin
|
|
|
284
348
|
// timeout
|
|
285
349
|
// 0 second
|
|
286
350
|
|
|
287
|
-
|
|
288
351
|
@Test
|
|
289
352
|
public void testUserPasswordAndPutToUserDirectoryRoot()
|
|
290
353
|
{
|
|
@@ -318,7 +381,6 @@ public class TestSftpFileOutputPlugin
|
|
|
318
381
|
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
|
319
382
|
testFolder.getRoot().getAbsolutePath(),
|
|
320
383
|
pathPrefix));
|
|
321
|
-
|
|
322
384
|
}
|
|
323
385
|
|
|
324
386
|
@Test
|
|
@@ -358,6 +420,60 @@ public class TestSftpFileOutputPlugin
|
|
|
358
420
|
pathPrefix));
|
|
359
421
|
}
|
|
360
422
|
|
|
423
|
+
@Test
|
|
424
|
+
public void testUserSecretKeyFileWithProxy()
|
|
425
|
+
{
|
|
426
|
+
HttpProxyServer proxyServer = null;
|
|
427
|
+
try {
|
|
428
|
+
proxyServer = createProxyServer(PROXY_PORT);
|
|
429
|
+
|
|
430
|
+
// setting embulk config
|
|
431
|
+
final String pathPrefix = "/test/testUserPassword";
|
|
432
|
+
String configYaml = "" +
|
|
433
|
+
"type: sftp\n" +
|
|
434
|
+
"host: " + HOST + "\n" +
|
|
435
|
+
"port: " + PORT + "\n" +
|
|
436
|
+
"user: " + USERNAME + "\n" +
|
|
437
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
|
438
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
|
439
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
|
440
|
+
"file_ext: txt\n" +
|
|
441
|
+
"proxy: \n" +
|
|
442
|
+
" type: http\n" +
|
|
443
|
+
" host: " + PROXY_HOST + "\n" +
|
|
444
|
+
" port: " + PROXY_PORT + " \n" +
|
|
445
|
+
" user: " + USERNAME + "\n" +
|
|
446
|
+
" password: " + PASSWORD + "\n" +
|
|
447
|
+
" command: \n" +
|
|
448
|
+
"formatter:\n" +
|
|
449
|
+
" type: csv\n" +
|
|
450
|
+
" newline: CRLF\n" +
|
|
451
|
+
" newline_in_field: LF\n" +
|
|
452
|
+
" header_line: true\n" +
|
|
453
|
+
" charset: UTF-8\n" +
|
|
454
|
+
" quote_policy: NONE\n" +
|
|
455
|
+
" quote: \"\\\"\"\n" +
|
|
456
|
+
" escape: \"\\\\\"\n" +
|
|
457
|
+
" null_string: \"\"\n" +
|
|
458
|
+
" default_timezone: 'UTC'";
|
|
459
|
+
|
|
460
|
+
// runner.transaction -> ...
|
|
461
|
+
run(configYaml, Optional.<Integer>absent());
|
|
462
|
+
|
|
463
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
|
464
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
|
465
|
+
|
|
466
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
|
467
|
+
testFolder.getRoot().getAbsolutePath(),
|
|
468
|
+
pathPrefix));
|
|
469
|
+
}
|
|
470
|
+
finally {
|
|
471
|
+
if (proxyServer != null) {
|
|
472
|
+
proxyServer.stop();
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
361
477
|
@Test
|
|
362
478
|
public void testTimeout()
|
|
363
479
|
{
|
|
@@ -393,4 +509,30 @@ public class TestSftpFileOutputPlugin
|
|
|
393
509
|
// runner.transaction -> ...
|
|
394
510
|
run(configYaml, Optional.of(60)); // sleep 1 minute while processing
|
|
395
511
|
}
|
|
512
|
+
|
|
513
|
+
@Test
|
|
514
|
+
public void testProxyType()
|
|
515
|
+
{
|
|
516
|
+
// test valueOf()
|
|
517
|
+
assertEquals("http", ProxyTask.ProxyType.valueOf("HTTP").toString());
|
|
518
|
+
assertEquals("socks", ProxyTask.ProxyType.valueOf("SOCKS").toString());
|
|
519
|
+
assertEquals("stream", ProxyTask.ProxyType.valueOf("STREAM").toString());
|
|
520
|
+
try {
|
|
521
|
+
ProxyTask.ProxyType.valueOf("non-existing-type");
|
|
522
|
+
}
|
|
523
|
+
catch (Exception e) {
|
|
524
|
+
assertEquals(IllegalArgumentException.class, e.getClass());
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
// test fromString
|
|
528
|
+
assertEquals(ProxyTask.ProxyType.HTTP, ProxyTask.ProxyType.fromString("http"));
|
|
529
|
+
assertEquals(ProxyTask.ProxyType.SOCKS, ProxyTask.ProxyType.fromString("socks"));
|
|
530
|
+
assertEquals(ProxyTask.ProxyType.STREAM, ProxyTask.ProxyType.fromString("stream"));
|
|
531
|
+
try {
|
|
532
|
+
ProxyTask.ProxyType.fromString("non-existing-type");
|
|
533
|
+
}
|
|
534
|
+
catch (Exception e) {
|
|
535
|
+
assertEquals(ConfigException.class, e.getClass());
|
|
536
|
+
}
|
|
537
|
+
}
|
|
396
538
|
}
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: embulk-output-sftp
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.0.
|
|
4
|
+
version: 0.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Civitaspo
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2016-03-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -47,9 +47,12 @@ extra_rdoc_files: []
|
|
|
47
47
|
files:
|
|
48
48
|
- .gitignore
|
|
49
49
|
- .travis.yml
|
|
50
|
+
- CHANGELOG.md
|
|
50
51
|
- LICENSE.txt
|
|
51
52
|
- README.md
|
|
52
53
|
- build.gradle
|
|
54
|
+
- config/checkstyle/checkstyle.xml
|
|
55
|
+
- config/checkstyle/default.xml
|
|
53
56
|
- example/data.csv
|
|
54
57
|
- example/sample.yml
|
|
55
58
|
- gradle/wrapper/gradle-wrapper.jar
|
|
@@ -57,6 +60,7 @@ files:
|
|
|
57
60
|
- gradlew
|
|
58
61
|
- gradlew.bat
|
|
59
62
|
- lib/embulk/output/sftp.rb
|
|
63
|
+
- src/main/java/org/embulk/output/sftp/ProxyTask.java
|
|
60
64
|
- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
|
|
61
65
|
- src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
|
|
62
66
|
- src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
|
|
@@ -64,7 +68,7 @@ files:
|
|
|
64
68
|
- src/test/resources/id_rsa.pub
|
|
65
69
|
- classpath/commons-logging-1.2.jar
|
|
66
70
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
|
67
|
-
- classpath/embulk-output-sftp-0.0.
|
|
71
|
+
- classpath/embulk-output-sftp-0.0.5.jar
|
|
68
72
|
- classpath/jsch-0.1.53.jar
|
|
69
73
|
homepage: https://github.com/civitaspo/embulk-output-sftp
|
|
70
74
|
licenses:
|
|
Binary file
|