embulk-input-sftp 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 72aaee2da1604f1a298e13d37ab74be174cddb5f
4
+ data.tar.gz: 3c062e178443efd8b44874eb9cb2d446385d010b
5
+ SHA512:
6
+ metadata.gz: d21a1f92aae66016ea648dc94e2f757e7bc3d968bda4953f35623307f649a49bde0d8b0bbbc84b79cedfb25c35e069f7b6b30134a3959d9211baad7581cb3c59
7
+ data.tar.gz: 67c3b0a1c2b054093f61aa5ca0946c9fed2f74a0f223e0ee60c44737fa33cbbd5fd90469f3b5baafc62eb45d74c19e504cd87c498a3ebdf8d3bb1dc7a8e5967f
@@ -0,0 +1,14 @@
1
+ target/
2
+ build/
3
+ pkg/
4
+ *.iml
5
+ *~
6
+ ._*
7
+ .idea
8
+ tmp/
9
+ vendor/
10
+ /classpath/
11
+ /.bundle
12
+ .yardoc
13
+ /embulk-*.jar
14
+ /.gradle
@@ -0,0 +1,126 @@
1
+ # SFTP file input plugin for Embulk
2
+
3
+ Reads files stored on remote server using SFTP
4
+
5
+ ## Overview
6
+
7
+ * **Plugin type**: file input
8
+ * **Resume supported**: yes
9
+ * **Cleanup supported**: yes
10
+
11
+ ## Configuration
12
+
13
+ - **host**: (string, required)
14
+ - **port**: (string, default: `22`)
15
+ - **user**: (string, required)
16
+ - **password**: (string, default: `null`)
17
+ - **secret_key_file**: (string, default: `null`)
18
+ - **secret_key_passphrase**: (string, default: `""`)
19
+ - **user_directory_is_root**: (boolean, default: `true`)
20
+ - **timeout**: sftp connection timeout seconds (integer, default: `600`)
21
+ - **path_prefix**: Prefix of output paths (string, required)
22
+ - **file_ext**: Extension of output files (string, required)
23
+ - **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
24
+
25
+ ### Proxy configuration
26
+
27
+ - **proxy**:
28
+ - **type**: (string(http | socks | stream), required, default: `null`)
29
+ - **http**: use HTTP Proxy
30
+ - **socks**: use SOCKS Proxy
31
+ - **stream**: Connects to the SFTP server through a remote host reached by SSH
32
+ - **host**: (string, required)
33
+ - **port**: (int, default: `22`)
34
+ - **user**: (string, optional)
35
+ - **password**: (string, optional, default: `null`)
36
+ - **command**: (string, optional)
37
+
38
+ ### Example
39
+
40
+ ```yaml
41
+ in:
42
+ type: sftp
43
+ host: 127.0.0.1
44
+ port: 22
45
+ user: embulk
46
+ secret_key_file: /Users/embulk/.ssh/id_rsa
47
+ secret_key_passphrase: secret_pass
48
+ user_directory_is_root: false
49
+ timeout: 600
50
+ path_prefix: /data/sftp
51
+ ```
52
+
53
+ With proxy
54
+ ```yaml
55
+ in:
56
+ type: sftp
57
+ host: 127.0.0.1
58
+ port: 22
59
+ user: embulk
60
+ secret_key_file: /Users/embulk/.ssh/id_rsa
61
+ secret_key_passphrase: secret_pass
62
+ user_directory_is_root: false
63
+ timeout: 600
64
+ path_prefix: /data/sftp
65
+ proxy:
66
+ type: http
67
+ host: proxy_host
68
+ port: 8080
69
+ user: proxy_user
70
+ password: proxy_secret_pass
71
+ command:
72
+ ```
73
+
74
+ ## Proxy settings
75
+
76
+ ### Example
77
+ ```yaml
78
+ in:
79
+ type: sftp
80
+ host: 127.0.0.1
81
+ port: 22
82
+ user: embulk
83
+ secret_key_file: /Users/embulk/.ssh/id_rsa
84
+ secret_key_passphrase: secret_pass
85
+ user_directory_is_root: false
86
+ timeout: 600
87
+ path_prefix: /data/sftp
88
+ ```
89
+
90
+ ### Secret Keyfile configuration
91
+
92
+ Please set path of secret_key_file as follows.
93
+ ```yaml
94
+ in:
95
+ type: sftp
96
+ ...
97
+ secret_key_file: /path/to/id_rsa
98
+ ...
99
+ ```
100
+
101
+ You can also embed contents of secret_key_file at config.yml.
102
+ ```yaml
103
+ in:
104
+ type: sftp
105
+ ...
106
+ secret_key_file:
107
+ content |
108
+ -----BEGIN RSA PRIVATE KEY-----
109
+ ABCDEFG...
110
+ HIJKLMN...
111
+ OPQRSTU...
112
+ -----END RSA PRIVATE KEY-----
113
+ ...
114
+ ```
115
+
116
+ ## Build
117
+
118
+ ```
119
+ $ ./gradlew gem # -t to watch change of files and rebuild continuously
120
+ ```
121
+
122
+ ## Test
123
+
124
+ ```
125
+ $ ./gradlew test # -t to watch change of files and rebuild continuously
126
+ ```
@@ -0,0 +1,95 @@
1
+ plugins {
2
+ id "com.jfrog.bintray" version "1.1"
3
+ id "com.github.jruby-gradle.base" version "0.1.5"
4
+ id "java"
5
+ id "checkstyle"
6
+ }
7
+ import com.github.jrubygradle.JRubyExec
8
+ repositories {
9
+ mavenCentral()
10
+ jcenter()
11
+ }
12
+ configurations {
13
+ provided
14
+ }
15
+
16
+ version = "0.1.0"
17
+
18
+ sourceCompatibility = 1.7
19
+ targetCompatibility = 1.7
20
+
21
+ dependencies {
22
+ compile "org.embulk:embulk-core:0.8.6"
23
+ provided "org.embulk:embulk-core:0.8.6"
24
+ compile "org.apache.commons:commons-vfs2:2.+"
25
+ compile "org.apache.commons:commons-io:1.3.2"
26
+ compile "com.jcraft:jsch:0.1.53"
27
+ testCompile "junit:junit:4.+"
28
+ testCompile "org.embulk:embulk-core:0.8.6:tests"
29
+ testCompile "org.embulk:embulk-standards:0.8.6"
30
+ testCompile "org.apache.sshd:apache-sshd:1.1.0"
31
+ testCompile "org.littleshoot:littleproxy:1.1.0-beta1"
32
+ testCompile "io.netty:netty-all:4.0.34.Final"
33
+ }
34
+
35
+ task classpath(type: Copy, dependsOn: ["jar"]) {
36
+ doFirst { file("classpath").deleteDir() }
37
+ from (configurations.runtime - configurations.provided + files(jar.archivePath))
38
+ into "classpath"
39
+ }
40
+ clean { delete "classpath" }
41
+
42
+ checkstyle {
43
+ configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
44
+ toolVersion = '6.14.1'
45
+ }
46
+ checkstyleMain {
47
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
48
+ ignoreFailures = true
49
+ }
50
+ checkstyleTest {
51
+ configFile = file("${project.rootDir}/config/checkstyle/default.xml")
52
+ ignoreFailures = true
53
+ }
54
+ task checkstyle(type: Checkstyle) {
55
+ classpath = sourceSets.main.output + sourceSets.test.output
56
+ source = sourceSets.main.allJava + sourceSets.test.allJava
57
+ }
58
+
59
+ task gem(type: JRubyExec, dependsOn: ["gemspec", "classpath"]) {
60
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
61
+ script "build/gemspec"
62
+ doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
63
+ }
64
+
65
+ task gemPush(type: JRubyExec, dependsOn: ["gem"]) {
66
+ jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "push"
67
+ script "pkg/${project.name}-${project.version}.gem"
68
+ }
69
+
70
+ task "package"(dependsOn: ["gemspec", "classpath"]) << {
71
+ println "> Build succeeded."
72
+ println "> You can run embulk with '-L ${file(".").absolutePath}' argument."
73
+ }
74
+
75
+ task gemspec << { file("build/gemspec").write($/
76
+ Gem::Specification.new do |spec|
77
+ spec.name = "${project.name}"
78
+ spec.version = "${project.version}"
79
+ spec.authors = ["Satoshi Akama"]
80
+ spec.summary = %[SFTP file input plugin for Embulk]
81
+ spec.description = %[Reads files stored on remote server using SFTP.]
82
+ spec.email = ["satoshiakama@gmail.com"]
83
+ spec.licenses = ["Apache-2.0"]
84
+ spec.homepage = "https://github.com/sakama/embulk-input-sftp"
85
+
86
+ spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
87
+ spec.test_files = spec.files.grep(%r"^(test|spec)/")
88
+ spec.require_paths = ["lib"]
89
+
90
+ spec.add_development_dependency 'bundler', ['~> 1.0']
91
+ spec.add_development_dependency 'rake', ['>= 10.0']
92
+ end
93
+ /$)
94
+ }
95
+ clean { delete "${project.name}.gemspec" }
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>