embulk-input-remote 0.3.0 → 0.3.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (22) hide show
  1. checksums.yaml +4 -4
  2. data/.circleci/config.yml +31 -0
  3. data/.gitignore +7 -0
  4. data/build.gradle +23 -50
  5. data/classpath/annotations-13.0.jar +0 -0
  6. data/classpath/embulk-input-remote-0.3.1.jar +0 -0
  7. data/classpath/kotlin-stdlib-1.1.1.jar +0 -0
  8. data/config/checkstyle/checkstyle.xml +128 -0
  9. data/config/checkstyle/default.xml +108 -0
  10. data/gradle.properties +1 -0
  11. data/lib/embulk/input/remote.rb +1 -1
  12. data/src/main/kotlin/org/embulk/input/{RemoteFileInputPlugin.kt → remote/RemoteFileInputPlugin.kt} +69 -76
  13. data/src/main/kotlin/org/embulk/input/remote/SSHClient.kt +22 -21
  14. data/src/main/kotlin/org/embulk/input/remote/extensions.kt +11 -0
  15. data/src/test/kotlin/org/embulk/input/remote/TestRemoteFileInputPlugin.kt +186 -0
  16. metadata +15 -12
  17. data/.travis.yml +0 -21
  18. data/classpath/embulk-input-remote-0.3.0.jar +0 -0
  19. data/classpath/kotlin-runtime-1.0.6.jar +0 -0
  20. data/classpath/kotlin-stdlib-1.0.6.jar +0 -0
  21. data/settings.gradle +0 -3
  22. data/src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt +0 -201
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: fe5d5ee6a70519b616bc61b9601d781f14510024
4
- data.tar.gz: 2bbe3b3615fd7bc8e29be25cb4755294ad8949e2
3
+ metadata.gz: 1d0aeb7be6efa2a882d28d13dc4a41c4d249042b
4
+ data.tar.gz: 7ffcbd5214290516ac430dea0149378b5b430a3c
5
5
  SHA512:
6
- metadata.gz: 0aa4e830ab9deaa103c9487797a9f4ccc4050811fd939f92789b6232c9a082b9014affc75931220d0eb52ced9cc4c4bc8e8c594315d47dc16e85153a38af1ec3
7
- data.tar.gz: b5a495f9d4f84bdc1dc1490465f8d84d2e8b5e3db308a96f3a909d7b16dc26fb5ad59263be6c74dd94b99f8681497e9614d4f3f844e2602086d810bdfe6c29f0
6
+ metadata.gz: c2de2f136d2d4e521227c4551c299862e8554b999ba213650aee2dc53bab091cd5201cb72808985d07508c987df2ca14c426ebffdc1153063e8127023c0c1b60
7
+ data.tar.gz: 0f44d0e50cb93413c6d3afa0650695161a1fb4a5806e28665e29c9a566fe35f3feda6101d9df33d314b339b2b96647003adefa9fbf948b4ed158f6d0339eff0f
data/.circleci/config.yml ADDED
@@ -0,0 +1,31 @@
1
+ version: 2
2
+ jobs:
3
+ build:
4
+ machine:
5
+ enabled: true
6
+ working_directory: ~/embulk-input-remote
7
+ steps:
8
+ - checkout
9
+
10
+ - run: ssh-keygen -t ecdsa -f id_rsa_test -N ''
11
+ - run: docker-compose up -d
12
+ - run: docker-compose ps
13
+
14
+ - restore_cache:
15
+ key: embulk-input-remote-{{ checksum "build.gradle" }}
16
+ - run: ./gradlew checkstyle
17
+ - run: ./gradlew check --full-stacktrace
18
+ - save_cache:
19
+ paths:
20
+ - "~/.gradle"
21
+ key: embulk-input-remote-{{ checksum "build.gradle" }}
22
+
23
+ - deploy:
24
+ name: Push Gem to RubyGems.org and bump up
25
+ command: |
26
+ if [ "${CIRCLE_BRANCH}" == "release" ]; then
27
+ curl -f -u $RUBYGEMS_USER:$RUBYGEMS_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials; chmod 0600 ~/.gem/credentials
28
+ git checkout master
29
+ git reset --hard origin/master
30
+ ./gradlew release -Prelease.useAutomaticVersion=true
31
+ fi
data/.gitignore CHANGED
@@ -1,7 +1,14 @@
1
1
  *~
2
2
  /pkg/
3
3
  /tmp/
4
+ *.gemspec
4
5
  .gradle/
5
6
  /classpath/
6
7
  build/
7
8
  .idea
9
+ /.settings/
10
+ /.metadata/
11
+ .classpath
12
+ .project
13
+ id_rsa_test
14
+ id_rsa_test.pub
data/build.gradle CHANGED
@@ -1,80 +1,53 @@
1
1
  buildscript {
2
- ext.kotlin_version = '1.0.6'
2
+ ext.kotlinVersion = '1.1.1'
3
3
  repositories {
4
4
  mavenCentral()
5
+ jcenter()
6
+ maven { url 'http://kamatama41.github.com/maven-repository/repository' }
5
7
  }
6
8
  dependencies {
7
- classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
9
+ classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlinVersion"
10
+ classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.4"
11
+ classpath "net.researchgate:gradle-release:2.5.0"
8
12
  }
9
13
  }
10
14
 
11
- plugins {
12
- id "com.jfrog.bintray" version "1.1"
13
- id "com.github.jruby-gradle.base" version "0.1.5"
14
- id "java"
15
- }
16
15
  apply plugin: "kotlin"
16
+ apply plugin: "com.github.kamatama41.embulk"
17
+ apply plugin: "net.researchgate.release"
17
18
 
18
19
  compileJava {
19
20
  options.compilerArgs = ['-Xlint:all']
20
21
  }
21
22
 
22
- import com.github.jrubygradle.JRubyExec
23
23
  repositories {
24
24
  mavenCentral()
25
- jcenter()
26
- maven { url 'http://kamatama41.github.com/embulk-test-helpers/repository' }
27
- }
28
- configurations {
29
- provided
25
+ maven { url 'http://kamatama41.github.com/maven-repository/repository' }
30
26
  }
31
27
 
32
- version = "0.3.0"
33
28
  sourceCompatibility = 1.7
34
29
  targetCompatibility = 1.7
35
30
 
36
31
  dependencies {
37
- compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
38
- compile "org.embulk:embulk-core:0.8.16"
32
+ compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlinVersion"
39
33
  compile "com.hierynomus:sshj:0.19.1"
40
34
  compile "com.jcraft:jzlib:1.1.3"
41
- provided "org.embulk:embulk-core:0.8.16"
42
- testCompile 'com.kamatama41:embulk-test-helpers:0.1.2'
35
+ testCompile "com.github.kamatama41:embulk-test-helpers:0.3.2"
43
36
  testCompile "com.github.docker-java:docker-java:3.0.7"
44
- // Uncomment when using local embulk-test-helpers (and settings.gradle as well)
45
- //testCompile project(':embulk-test-helpers')
46
- }
47
-
48
- task classpath(type: Copy, dependsOn: ["jar"]) {
49
- doFirst { file("classpath").deleteDir() }
50
- from (configurations.runtime - configurations.provided + files(jar.archivePath))
51
- into "classpath"
52
37
  }
53
- clean { delete 'classpath' }
54
38
 
55
- task gem(type: JRubyExec, dependsOn: ["build", "gemspec", "classpath"]) {
56
- jrubyArgs "-rrubygems/gem_runner", "-eGem::GemRunner.new.run(ARGV)", "build"
57
- script "build/gemspec"
58
- doLast { ant.move(file: "${project.name}-${project.version}.gem", todir: "pkg") }
39
+ embulk {
40
+ version = "0.8.18"
41
+ category = "file-input"
42
+ name = "remote"
43
+ authors = ["Shinichi Ishimura"]
44
+ email = "shiketaudonko41@gmail.com"
45
+ description = "Reads files stored on Remote hosts by SCP."
46
+ licenses = ["MIT"]
47
+ homepage = "https://github.com/kamatama41/embulk-input-remote"
59
48
  }
60
49
 
61
- task gemspec << { file("build/gemspec").write($/
62
- Gem::Specification.new do |spec|
63
- spec.name = "${project.name}"
64
- spec.version = "${project.version}"
65
- spec.authors = ["Shinichi Ishimura"]
66
- spec.summary = %[Remote file input plugin for Embulk]
67
- spec.description = %[Reads files stored on Remote hosts by SCP.]
68
- spec.email = ["shiketaudonko41@gmail.com"]
69
- spec.licenses = ["MIT"]
70
- spec.homepage = "https://github.com/kamatama41/embulk-input-remote"
71
-
72
- spec.files = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
73
- spec.test_files = spec.files.grep(%r"^(test|spec)/")
74
- spec.require_paths = ["lib"]
75
-
76
- spec.add_development_dependency 'bundler', ['~> 1.0']
77
- spec.add_development_dependency 'rake', ['>= 10.0']
78
- end
79
- /$)
50
+ release {
51
+ git { requireBranch = 'master' }
80
52
  }
53
+ afterReleaseBuild.dependsOn gemPush
data/classpath/annotations-13.0.jar ADDED
Binary file
data/classpath/kotlin-stdlib-1.1.1.jar ADDED
Binary file
data/config/checkstyle/checkstyle.xml ADDED
@@ -0,0 +1,128 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <module name="Checker">
6
+ <!-- https://github.com/facebook/presto/blob/master/src/checkstyle/checks.xml -->
7
+ <module name="FileTabCharacter"/>
8
+ <module name="NewlineAtEndOfFile">
9
+ <property name="lineSeparator" value="lf"/>
10
+ </module>
11
+ <module name="RegexpMultiline">
12
+ <property name="format" value="\r"/>
13
+ <property name="message" value="Line contains carriage return"/>
14
+ </module>
15
+ <module name="RegexpMultiline">
16
+ <property name="format" value=" \n"/>
17
+ <property name="message" value="Line has trailing whitespace"/>
18
+ </module>
19
+ <module name="RegexpMultiline">
20
+ <property name="format" value="\{\n\n"/>
21
+ <property name="message" value="Blank line after opening brace"/>
22
+ </module>
23
+ <module name="RegexpMultiline">
24
+ <property name="format" value="\n\n\s*\}"/>
25
+ <property name="message" value="Blank line before closing brace"/>
26
+ </module>
27
+ <module name="RegexpMultiline">
28
+ <property name="format" value="\n\n\n"/>
29
+ <property name="message" value="Multiple consecutive blank lines"/>
30
+ </module>
31
+ <module name="RegexpMultiline">
32
+ <property name="format" value="\n\n\Z"/>
33
+ <property name="message" value="Blank line before end of file"/>
34
+ </module>
35
+ <module name="RegexpMultiline">
36
+ <property name="format" value="Preconditions\.checkNotNull"/>
37
+ <property name="message" value="Use of checkNotNull"/>
38
+ </module>
39
+
40
+ <module name="TreeWalker">
41
+ <module name="EmptyBlock">
42
+ <property name="option" value="text"/>
43
+ <property name="tokens" value="
44
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
45
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
46
+ </module>
47
+ <module name="EmptyStatement"/>
48
+ <module name="EmptyForInitializerPad"/>
49
+ <module name="EmptyForIteratorPad">
50
+ <property name="option" value="space"/>
51
+ </module>
52
+ <module name="MethodParamPad">
53
+ <property name="allowLineBreaks" value="true"/>
54
+ <property name="option" value="nospace"/>
55
+ </module>
56
+ <module name="ParenPad"/>
57
+ <module name="TypecastParenPad"/>
58
+ <module name="NeedBraces"/>
59
+ <module name="LeftCurly">
60
+ <property name="option" value="nl"/>
61
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
62
+ </module>
63
+ <module name="LeftCurly">
64
+ <property name="option" value="eol"/>
65
+ <property name="tokens" value="
66
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
67
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
68
+ </module>
69
+ <module name="RightCurly">
70
+ <property name="option" value="alone"/>
71
+ </module>
72
+ <module name="GenericWhitespace"/>
73
+ <module name="WhitespaceAfter"/>
74
+ <module name="NoWhitespaceBefore"/>
75
+
76
+ <module name="UpperEll"/>
77
+ <module name="DefaultComesLast"/>
78
+ <module name="ArrayTypeStyle"/>
79
+ <module name="MultipleVariableDeclarations"/>
80
+ <module name="ModifierOrder"/>
81
+ <module name="OneStatementPerLine"/>
82
+ <module name="StringLiteralEquality"/>
83
+ <module name="MutableException"/>
84
+ <module name="EqualsHashCode"/>
85
+ <module name="InnerAssignment"/>
86
+ <module name="InterfaceIsType"/>
87
+ <module name="HideUtilityClassConstructor"/>
88
+
89
+ <module name="MemberName"/>
90
+ <module name="LocalVariableName"/>
91
+ <module name="LocalFinalVariableName"/>
92
+ <module name="TypeName"/>
93
+ <module name="PackageName"/>
94
+ <module name="ParameterName"/>
95
+ <module name="StaticVariableName"/>
96
+ <module name="ClassTypeParameterName">
97
+ <property name="format" value="^[A-Z][0-9]?$"/>
98
+ </module>
99
+ <module name="MethodTypeParameterName">
100
+ <property name="format" value="^[A-Z][0-9]?$"/>
101
+ </module>
102
+
103
+ <module name="AvoidStarImport"/>
104
+ <module name="RedundantImport"/>
105
+ <module name="UnusedImports"/>
106
+ <module name="ImportOrder">
107
+ <property name="groups" value="*,javax,java"/>
108
+ <property name="separated" value="true"/>
109
+ <property name="option" value="bottom"/>
110
+ <property name="sortStaticImportsAlphabetically" value="true"/>
111
+ </module>
112
+
113
+ <module name="WhitespaceAround">
114
+ <property name="allowEmptyConstructors" value="true"/>
115
+ <property name="allowEmptyMethods" value="true"/>
116
+ <property name="ignoreEnhancedForColon" value="false"/>
117
+ <property name="tokens" value="
118
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
119
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
120
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
121
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
122
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
123
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
124
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
125
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
126
+ </module>
127
+ </module>
128
+ </module>
data/config/checkstyle/default.xml ADDED
@@ -0,0 +1,108 @@
1
+ <?xml version="1.0" encoding="UTF-8"?>
2
+ <!DOCTYPE module PUBLIC
3
+ "-//Puppy Crawl//DTD Check Configuration 1.3//EN"
4
+ "http://www.puppycrawl.com/dtds/configuration_1_3.dtd">
5
+ <!--
6
+ This is a subset of ./checkstyle.xml which allows some loose styles
7
+ -->
8
+ <module name="Checker">
9
+ <module name="FileTabCharacter"/>
10
+ <module name="NewlineAtEndOfFile">
11
+ <property name="lineSeparator" value="lf"/>
12
+ </module>
13
+ <module name="RegexpMultiline">
14
+ <property name="format" value="\r"/>
15
+ <property name="message" value="Line contains carriage return"/>
16
+ </module>
17
+ <module name="RegexpMultiline">
18
+ <property name="format" value=" \n"/>
19
+ <property name="message" value="Line has trailing whitespace"/>
20
+ </module>
21
+ <module name="RegexpMultiline">
22
+ <property name="format" value="\n\n\n"/>
23
+ <property name="message" value="Multiple consecutive blank lines"/>
24
+ </module>
25
+ <module name="RegexpMultiline">
26
+ <property name="format" value="\n\n\Z"/>
27
+ <property name="message" value="Blank line before end of file"/>
28
+ </module>
29
+
30
+ <module name="TreeWalker">
31
+ <module name="EmptyBlock">
32
+ <property name="option" value="text"/>
33
+ <property name="tokens" value="
34
+ LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_IF,
35
+ LITERAL_FOR, LITERAL_TRY, LITERAL_WHILE, INSTANCE_INIT, STATIC_INIT"/>
36
+ </module>
37
+ <module name="EmptyStatement"/>
38
+ <module name="EmptyForInitializerPad"/>
39
+ <module name="EmptyForIteratorPad">
40
+ <property name="option" value="space"/>
41
+ </module>
42
+ <module name="MethodParamPad">
43
+ <property name="allowLineBreaks" value="true"/>
44
+ <property name="option" value="nospace"/>
45
+ </module>
46
+ <module name="ParenPad"/>
47
+ <module name="TypecastParenPad"/>
48
+ <module name="NeedBraces"/>
49
+ <module name="LeftCurly">
50
+ <property name="option" value="nl"/>
51
+ <property name="tokens" value="CLASS_DEF, CTOR_DEF, INTERFACE_DEF, METHOD_DEF"/>
52
+ </module>
53
+ <module name="LeftCurly">
54
+ <property name="option" value="eol"/>
55
+ <property name="tokens" value="
56
+ LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE, LITERAL_FINALLY, LITERAL_FOR,
57
+ LITERAL_IF, LITERAL_SWITCH, LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE"/>
58
+ </module>
59
+ <module name="RightCurly">
60
+ <property name="option" value="alone"/>
61
+ </module>
62
+ <module name="GenericWhitespace"/>
63
+ <module name="WhitespaceAfter"/>
64
+ <module name="NoWhitespaceBefore"/>
65
+
66
+ <module name="UpperEll"/>
67
+ <module name="DefaultComesLast"/>
68
+ <module name="ArrayTypeStyle"/>
69
+ <module name="MultipleVariableDeclarations"/>
70
+ <module name="ModifierOrder"/>
71
+ <module name="OneStatementPerLine"/>
72
+ <module name="StringLiteralEquality"/>
73
+ <module name="MutableException"/>
74
+ <module name="EqualsHashCode"/>
75
+ <module name="InnerAssignment"/>
76
+ <module name="InterfaceIsType"/>
77
+ <module name="HideUtilityClassConstructor"/>
78
+
79
+ <module name="MemberName"/>
80
+ <module name="LocalVariableName"/>
81
+ <module name="LocalFinalVariableName"/>
82
+ <module name="TypeName"/>
83
+ <module name="PackageName"/>
84
+ <module name="ParameterName"/>
85
+ <module name="StaticVariableName"/>
86
+ <module name="ClassTypeParameterName">
87
+ <property name="format" value="^[A-Z][0-9]?$"/>
88
+ </module>
89
+ <module name="MethodTypeParameterName">
90
+ <property name="format" value="^[A-Z][0-9]?$"/>
91
+ </module>
92
+
93
+ <module name="WhitespaceAround">
94
+ <property name="allowEmptyConstructors" value="true"/>
95
+ <property name="allowEmptyMethods" value="true"/>
96
+ <property name="ignoreEnhancedForColon" value="false"/>
97
+ <property name="tokens" value="
98
+ ASSIGN, BAND, BAND_ASSIGN, BOR, BOR_ASSIGN, BSR, BSR_ASSIGN,
99
+ BXOR, BXOR_ASSIGN, COLON, DIV, DIV_ASSIGN, EQUAL, GE, GT, LAND, LE,
100
+ LITERAL_ASSERT, LITERAL_CATCH, LITERAL_DO, LITERAL_ELSE,
101
+ LITERAL_FINALLY, LITERAL_FOR, LITERAL_IF, LITERAL_RETURN,
102
+ LITERAL_SYNCHRONIZED, LITERAL_TRY, LITERAL_WHILE,
103
+ LOR, LT, MINUS, MINUS_ASSIGN, MOD, MOD_ASSIGN, NOT_EQUAL,
104
+ PLUS, PLUS_ASSIGN, QUESTION, SL, SLIST, SL_ASSIGN, SR, SR_ASSIGN,
105
+ STAR, STAR_ASSIGN, TYPE_EXTENSION_AND"/>
106
+ </module>
107
+ </module>
108
+ </module>
data/gradle.properties ADDED
@@ -0,0 +1 @@
1
+ version=0.3.1
data/lib/embulk/input/remote.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  Embulk::JavaPlugin.register_input(
2
- "remote", "org.embulk.input.RemoteFileInputPlugin",
2
+ "remote", "org.embulk.input.remote.RemoteFileInputPlugin",
3
3
  File.expand_path('../../../../classpath', __FILE__))
data/src/main/kotlin/org/embulk/input/{RemoteFileInputPlugin.kt → remote/RemoteFileInputPlugin.kt} RENAMED
@@ -1,4 +1,4 @@
1
- package org.embulk.input
1
+ package org.embulk.input.remote
2
2
 
3
3
  import com.fasterxml.jackson.annotation.JsonProperty
4
4
  import com.google.common.base.Optional
@@ -10,7 +10,6 @@ import org.embulk.config.ConfigSource
10
10
  import org.embulk.config.Task
11
11
  import org.embulk.config.TaskReport
12
12
  import org.embulk.config.TaskSource
13
- import org.embulk.input.remote.SSHClient
14
13
  import org.embulk.spi.BufferAllocator
15
14
  import org.embulk.spi.Exec
16
15
  import org.embulk.spi.FileInputPlugin
@@ -23,81 +22,78 @@ import java.io.IOException
23
22
  import java.io.InputStream
24
23
  import java.io.InputStreamReader
25
24
 
26
-
27
25
  class RemoteFileInputPlugin : FileInputPlugin {
28
26
  interface PluginTask : Task {
29
- @Config("hosts")
30
- @ConfigDefault("[]")
31
- fun getHosts(): List<String>
32
-
33
- @Config("hosts_command")
34
- @ConfigDefault("null")
35
- fun getHostsCommand(): Optional<String>
27
+ @get:Config("hosts")
28
+ @get:ConfigDefault("[]")
29
+ val hosts: List<String>
36
30
 
37
- @Config("hosts_separator")
38
- @ConfigDefault("\" \"")
39
- fun getHostsSeparator(): String
31
+ @get:Config("hosts_command")
32
+ @get:ConfigDefault("null")
33
+ val hostsCommand: Optional<String>
40
34
 
41
- @Config("default_port")
42
- @ConfigDefault("22")
43
- fun getDefaultPort(): Int
35
+ @get:Config("hosts_separator")
36
+ @get:ConfigDefault("\" \"")
37
+ val hostsSeparator: String
44
38
 
45
- @Config("path")
46
- @ConfigDefault("\"\"")
47
- fun getPath(): String
39
+ @get:Config("default_port")
40
+ @get:ConfigDefault("22")
41
+ val defaultPort: Int
48
42
 
49
- @Config("path_command")
50
- @ConfigDefault("null")
51
- fun getPathCommand(): Optional<String>
43
+ @get:Config("path")
44
+ @get:ConfigDefault("\"\"")
45
+ val path: String
52
46
 
53
- @Config("auth")
54
- fun getAuthConfig(): AuthConfig
47
+ @get:Config("path_command")
48
+ @get:ConfigDefault("null")
49
+ val pathCommand: Optional<String>
55
50
 
56
- @Config("ignore_not_found_hosts")
57
- @ConfigDefault("false")
58
- fun getIgnoreNotFoundHosts(): Boolean
51
+ @get:Config("auth")
52
+ val authConfig: AuthConfig
59
53
 
60
- @Config("done_targets")
61
- @ConfigDefault("[]")
62
- fun getDoneTargets(): List<Target>
54
+ @get:Config("ignore_not_found_hosts")
55
+ @get:ConfigDefault("false")
56
+ val ignoreNotFoundHosts: Boolean
63
57
 
64
- fun getTargets(): List<Target>
58
+ @get:Config("done_targets")
59
+ @get:ConfigDefault("[]")
60
+ val doneTargets: List<Target>
65
61
 
66
- fun setTargets(targets: List<Target>)
62
+ var targets: List<Target>
67
63
 
68
- @ConfigInject
69
- fun getBufferAllocator(): BufferAllocator
64
+ @get:ConfigInject
65
+ val bufferAllocator: BufferAllocator
70
66
  }
71
67
 
72
68
  interface AuthConfig : Task {
73
- @Config("type")
74
- @ConfigDefault("\"public_key\"")
75
- fun getType(): String
69
+ @get:Config("type")
70
+ @get:ConfigDefault("\"public_key\"")
71
+ val type: String
76
72
 
77
- @Config("user")
78
- @ConfigDefault("null")
79
- fun getUser(): Optional<String>
73
+ @get:Config("user")
74
+ @get:ConfigDefault("null")
75
+ val user: Optional<String>
80
76
 
81
- @Config("key_path")
82
- @ConfigDefault("null")
83
- fun getKeyPath(): Optional<String>
77
+ @get:Config("key_path")
78
+ @get:ConfigDefault("null")
79
+ val keyPath: Optional<String>
84
80
 
85
- @Config("password")
86
- @ConfigDefault("null")
87
- fun getPassword(): Optional<String>
81
+ @get:Config("password")
82
+ @get:ConfigDefault("null")
83
+ val password: Optional<String>
88
84
 
89
- @Config("skip_host_key_verification")
90
- @ConfigDefault("false")
91
- fun getSkipHostKeyVerification(): Boolean
85
+ @get:Config("skip_host_key_verification")
86
+ @get:ConfigDefault("false")
87
+ val skipHostKeyVerification: Boolean
92
88
  }
93
89
 
94
- private val log = Exec.getLogger(javaClass)
90
+ private val log = getLogger()
95
91
 
96
92
  override fun transaction(config: ConfigSource, control: FileInputPlugin.Control): ConfigDiff {
97
- val task = config.loadConfig(PluginTask::class.java)
93
+ val task: PluginTask = config.loadConfig()
98
94
  val targets = listTargets(task)
99
95
  log.info("Loading targets $targets")
100
- task.setTargets(targets)
96
+ task.targets = targets
101
97
 
102
98
  // number of processors is same with number of targets
103
99
  val taskCount = targets.size
@@ -105,21 +101,21 @@ class RemoteFileInputPlugin : FileInputPlugin {
105
101
  }
106
102
 
107
103
  override fun resume(taskSource: TaskSource, taskCount: Int, control: FileInputPlugin.Control): ConfigDiff {
108
- val task = taskSource.loadTask(PluginTask::class.java)
104
+ val task: PluginTask = taskSource.loadTask()
109
105
 
110
106
  control.run(taskSource, taskCount)
111
107
 
112
- return Exec.newConfigDiff().set("done_targets", task.getTargets())
108
+ return Exec.newConfigDiff().set("done_targets", task.targets)
113
109
  }
114
110
 
115
111
  override fun cleanup(taskSource: TaskSource, taskCount: Int, successTaskReports: MutableList<TaskReport>) {
116
112
  }
117
113
 
118
114
  override fun open(taskSource: TaskSource, taskIndex: Int): TransactionalFileInput {
119
- val task = taskSource.loadTask(PluginTask::class.java)
120
- val target = task.getTargets()[taskIndex]
115
+ val task: PluginTask = taskSource.loadTask()
116
+ val target = task.targets[taskIndex]
121
117
 
122
- return object : InputStreamTransactionalFileInput(task.getBufferAllocator(), { download(target, task) }) {
118
+ return object : InputStreamTransactionalFileInput(task.bufferAllocator, { download(target, task) }) {
123
119
  override fun abort() {
124
120
  }
125
121
 
@@ -132,36 +128,34 @@ class RemoteFileInputPlugin : FileInputPlugin {
132
128
  private fun listTargets(task: PluginTask): List<Target> {
133
129
  val hosts = listHosts(task)
134
130
  val path = getPath(task)
135
- val doneTargets = task.getDoneTargets()
136
- val ignoreNotFoundHosts = task.getIgnoreNotFoundHosts()
131
+ val doneTargets = task.doneTargets
132
+ val ignoreNotFoundHosts = task.ignoreNotFoundHosts
137
133
 
138
134
  return hosts.map {
139
135
  val split = it.split(Regex(":"))
140
136
  val host = split[0]
141
- val port = if (split.size > 1) split[1].toInt() else task.getDefaultPort()
137
+ val port = if (split.size > 1) split[1].toInt() else task.defaultPort
142
138
  Target(host, port, path)
143
139
  }.filter {
144
140
  !doneTargets.contains(it)
145
141
  }.filter {
146
- !ignoreNotFoundHosts || {
147
- try {
148
- exists(it, task)
149
- } catch (e: IOException) {
150
- log.warn("failed to check the file exists. $it", e)
151
- false
152
- }
153
- }()
142
+ !ignoreNotFoundHosts || try {
143
+ exists(it, task)
144
+ } catch (e: IOException) {
145
+ log.warn("failed to check the file exists. $it", e)
146
+ false
147
+ }
154
148
  }
155
149
  }
156
150
 
157
151
  private fun listHosts(task: PluginTask): List<String> {
158
- return task.getHostsCommand().transform {
159
- execCommand(it).split(task.getHostsSeparator().toRegex())
160
- }.or(task.getHosts())
152
+ return task.hostsCommand.transform {
153
+ execCommand(it).split(task.hostsSeparator.toRegex())
154
+ }.or(task.hosts)
161
155
  }
162
156
 
163
157
  private fun getPath(task: PluginTask): String {
164
- return task.getPathCommand().transform { execCommand(it) }.or(task.getPath())
158
+ return task.pathCommand.transform { execCommand(it) }.or(task.path)
165
159
  }
166
160
 
167
161
  private fun execCommand(command: String?): String {
@@ -183,9 +177,9 @@ class RemoteFileInputPlugin : FileInputPlugin {
183
177
  }
184
178
 
185
179
  private fun exists(target: Target, task: PluginTask): Boolean {
186
- SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
180
+ SSHClient.connect(target.host, target.port, task.authConfig).use { client ->
187
181
  val checkCmd = "ls ${target.path}" // TODO: windows
188
- val timeout = 5/* second */
182
+ val timeout = 5 /* seconds */
189
183
  val commandResult = client.execCommand(checkCmd, timeout)
190
184
 
191
185
  if (commandResult.status != 0) {
@@ -197,7 +191,7 @@ class RemoteFileInputPlugin : FileInputPlugin {
197
191
  }
198
192
 
199
193
  private fun download(target: Target, task: PluginTask): InputStream {
200
- SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
194
+ SSHClient.connect(target.host, target.port, task.authConfig).use { client ->
201
195
  val stream = ByteArrayOutputStream()
202
196
  client.scpDownload(target.path, stream)
203
197
  return ByteArrayInputStream(stream.toByteArray())
@@ -208,7 +202,6 @@ class RemoteFileInputPlugin : FileInputPlugin {
208
202
  @JsonProperty("host") val host: String,
209
203
  @JsonProperty("port") val port: Int,
210
204
  @JsonProperty("path") val path: String) {
211
-
212
205
  override fun toString(): String {
213
206
  return "$host:$port:$path"
214
207
  }
data/src/main/kotlin/org/embulk/input/remote/SSHClient.kt CHANGED
@@ -1,47 +1,48 @@
1
1
  package org.embulk.input.remote
2
2
 
3
3
  import net.schmizz.sshj.DefaultConfig
4
+ import net.schmizz.sshj.SSHClient as SSHJ
4
5
  import net.schmizz.sshj.transport.verification.PromiscuousVerifier
5
6
  import net.schmizz.sshj.xfer.InMemoryDestFile
6
7
  import net.schmizz.sshj.xfer.LocalDestFile
7
- import org.embulk.input.RemoteFileInputPlugin
8
8
  import java.io.Closeable
9
9
  import java.io.InputStream
10
10
  import java.io.OutputStream
11
11
  import java.util.concurrent.TimeUnit
12
12
 
13
- class SSHClient private constructor(val client: net.schmizz.sshj.SSHClient) : Closeable {
13
+ class SSHClient private constructor(val client: SSHJ) : Closeable {
14
14
  companion object {
15
- @JvmStatic
16
- fun connect(
17
- host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig
18
- ): SSHClient {
19
- val client = SSHClient(net.schmizz.sshj.SSHClient(DefaultConfig()))
15
+ fun connect(host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig): SSHClient {
16
+ val client = SSHClient(SSHJ(DefaultConfig()))
20
17
  client.connectToHost(host, port, authConfig)
21
18
  return client
22
19
  }
23
20
  }
24
21
 
25
22
  private fun connectToHost(host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig) {
26
- if (authConfig.getSkipHostKeyVerification()) {
23
+ if (authConfig.skipHostKeyVerification) {
27
24
  client.addHostKeyVerifier(PromiscuousVerifier())
28
25
  }
29
26
  client.loadKnownHosts()
30
27
  client.connect(host, port)
31
28
 
32
- val type = authConfig.getType()
33
- val user = authConfig.getUser().or(System.getProperty("user.name"))
29
+ val type = authConfig.type
30
+ val user = authConfig.user.or(System.getProperty("user.name"))
34
31
 
35
- if ("password" == type) {
36
- client.authPassword(user, authConfig.getPassword().get())
37
- } else if ("public_key" == type) {
38
- authConfig.getKeyPath().transform {
39
- client.authPublickey(user, it)
40
- }.or {
41
- client.authPublickey(user)
32
+ when (type) {
33
+ "password" -> {
34
+ client.authPassword(user, authConfig.password.get())
35
+ }
36
+ "public_key" -> {
37
+ authConfig.keyPath.transform {
38
+ client.authPublickey(user, it)
39
+ }.or {
40
+ client.authPublickey(user)
41
+ }
42
+ }
43
+ else -> {
44
+ throw UnsupportedOperationException("Unsupported auth type : $type")
42
45
  }
43
- } else {
44
- throw UnsupportedOperationException("Unsupported auth type : " + type)
45
46
  }
46
47
  }
47
48
 
@@ -49,13 +50,13 @@ class SSHClient private constructor(val client: net.schmizz.sshj.SSHClient) : Cl
49
50
  client.startSession().use { session ->
50
51
  val cmd = session.exec(command)
51
52
  cmd.join(timeoutSecond.toLong(), TimeUnit.SECONDS)
52
- return CommandResult(cmd.exitStatus!!, cmd.inputStream)
53
+ return CommandResult(cmd.exitStatus, cmd.inputStream)
53
54
  }
54
55
  }
55
56
 
56
57
  fun scpDownload(path: String, stream: OutputStream) {
57
58
  client.useCompression()
58
- client.newSCPFileTransfer().download(path, object :InMemoryDestFile() {
59
+ client.newSCPFileTransfer().download(path, object : InMemoryDestFile() {
59
60
  override fun getOutputStream(): OutputStream {
60
61
  return stream
61
62
  }
data/src/main/kotlin/org/embulk/input/remote/extensions.kt ADDED
@@ -0,0 +1,11 @@
1
+ package org.embulk.input.remote
2
+
3
+ import org.embulk.config.ConfigSource
4
+ import org.embulk.config.TaskSource
5
+ import org.embulk.spi.Exec
6
+
7
+ inline fun <reified T : Any> ConfigSource.loadConfig() = loadConfig(T::class.java)!!
8
+
9
+ inline fun <reified T : Any> TaskSource.loadTask() = loadTask(T::class.java)!!
10
+
11
+ fun Any.getLogger() = Exec.getLogger(javaClass)!!
data/src/test/kotlin/org/embulk/input/remote/TestRemoteFileInputPlugin.kt ADDED
@@ -0,0 +1,186 @@
1
+ package org.embulk.input.remote
2
+
3
+ import ch.qos.logback.classic.Level
4
+ import ch.qos.logback.classic.Logger
5
+ import com.github.dockerjava.core.DockerClientBuilder
6
+ import org.embulk.config.ConfigSource
7
+ import org.embulk.test.EmbulkPluginTest
8
+ import org.embulk.test.TestOutputPlugin.Matcher.assertRecords
9
+ import org.embulk.test.configFromResource
10
+ import org.embulk.test.record
11
+ import org.embulk.test.registerPlugin
12
+ import org.embulk.test.set
13
+ import org.hamcrest.CoreMatchers.`is`
14
+ import org.hamcrest.MatcherAssert.assertThat
15
+ import org.junit.Before
16
+ import org.junit.Ignore
17
+ import org.junit.Test
18
+ import org.slf4j.LoggerFactory
19
+
20
+ class TestRemoteFileInputPlugin : EmbulkPluginTest() {
21
+ @Before fun setup() {
22
+ builder.registerPlugin(RemoteFileInputPlugin::class)
23
+
24
+ // Setup docker container
25
+ startContainer(CONTAINER_ID_HOST1)
26
+ startContainer(CONTAINER_ID_HOST2)
27
+
28
+ System.getenv("LOG_LEVEL")?.let {
29
+ // Set log level
30
+ val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger
31
+ rootLogger.level = Level.toLevel(it)
32
+ }
33
+ }
34
+
35
+ @Test fun loadFromRemote() {
36
+ runInput(baseConfig())
37
+ assertRecords(record(1, "user1"))
38
+ }
39
+
40
+ @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
41
+ @Test fun loadFromRemoteViaPublicKey() {
42
+ val keyPath = System.getenv("KEY_PATH") ?: "./id_rsa_test"
43
+
44
+ val publicKeyAuth = config().set("auth" to config().set(
45
+ "type" to "public_key",
46
+ "key_path" to keyPath
47
+ ))
48
+ runInput(baseConfig().merge(publicKeyAuth))
49
+
50
+ assertRecords(record(1, "user1"))
51
+ }
52
+
53
+ @Test fun testMultiHosts() {
54
+ val multiHosts = config()
55
+ .set("hosts", listOf("localhost:10022", "localhost:10023"))
56
+
57
+ // Run
58
+ runInput(baseConfig().merge(multiHosts))
59
+ assertRecords(
60
+ record(1, "user1"),
61
+ record(2, "user2")
62
+ )
63
+ }
64
+
65
+ @Test fun loadAllFilesInDirectory() {
66
+ val directoryPath = config().set("path", "/mount")
67
+
68
+ runInput(baseConfig().merge(directoryPath))
69
+ assertRecords(
70
+ record(1L, "user1"),
71
+ record(1L, "command_user1")
72
+ )
73
+ }
74
+
75
+ @Test fun testDefaultPort() {
76
+ val defaultPort = config().set(
77
+ "hosts" to listOf("localhost"),
78
+ "default_port" to 10022
79
+ )
80
+
81
+ runInput(baseConfig().merge(defaultPort))
82
+
83
+ assertRecords(record(1L, "user1"))
84
+ }
85
+
86
+ @Test fun testConfDiff() {
87
+ val host2Config = config().set("hosts", listOf("localhost:10023"))
88
+
89
+ // Run
90
+ val runResult = runInput(baseConfig().merge(host2Config))
91
+ assertRecords(record(2, "user2"))
92
+
93
+ // Re-run with additional host1
94
+ val multiHost = config().set("hosts", listOf("localhost:10022", "localhost:10023"))
95
+ runInput(baseConfig().merge(multiHost), runResult.configDiff)
96
+
97
+ assertRecords(record(1, "user1"))
98
+ }
99
+
100
+ @Test fun testResume() {
101
+ // Stop host2 temporarily
102
+ stopContainer(CONTAINER_ID_HOST2)
103
+
104
+ // Run (but will fail)
105
+ val multiHost = config().set("hosts", listOf("localhost:10022", "localhost:10023"))
106
+ val config = baseConfig().merge(multiHost)
107
+ var resumableResult = resume(config)
108
+
109
+ assertThat(resumableResult.isSuccessful, `is`(false))
110
+ assertRecords(record(1, "user1"))
111
+
112
+ // Start host2 again
113
+ startContainer(CONTAINER_ID_HOST2)
114
+
115
+ // Resume
116
+ resumableResult = resume(config, resumableResult.resumeState)
117
+
118
+ assertThat(resumableResult.isSuccessful, `is`(true))
119
+ assertRecords(record(2, "user2"))
120
+ }
121
+
122
+ @Test fun testIgnoreNotFoundHosts() {
123
+ val ignoreNotFoundHosts = config().set(
124
+ "hosts" to listOf("localhost:10022", "localhost:10023"),
125
+ "ignore_not_found_hosts" to true
126
+ )
127
+ val config = baseConfig().merge(ignoreNotFoundHosts)
128
+
129
+ // Stop host2
130
+ stopContainer(CONTAINER_ID_HOST2)
131
+
132
+ // Run (host2 will be ignored)
133
+ val resumableResult = resume(config)
134
+
135
+ assertThat<Boolean>(resumableResult.isSuccessful, `is`(true))
136
+ assertRecords(record(1, "user1"))
137
+ }
138
+
139
+ @Test fun testCommandOptions() {
140
+ val ignoreNotFoundHosts = config().set(
141
+ "hosts_command" to "./src/test/resources/script/hosts.sh",
142
+ "hosts_separator" to "\n",
143
+ "path_command" to "echo '/mount/test_command.csv'"
144
+ )
145
+ runInput(baseConfig().merge(ignoreNotFoundHosts))
146
+
147
+ assertRecords(
148
+ record(1, "command_user1"),
149
+ record(2, "command_user2")
150
+ )
151
+ }
152
+
153
+ //////////////////////////////
154
+ // Helpers
155
+ //////////////////////////////
156
+
157
+ private fun baseConfig(): ConfigSource {
158
+ return configFromResource("yaml/base.yml")
159
+ }
160
+
161
+ companion object DockerUtils {
162
+ private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
163
+ private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
164
+ private val dockerClient = DockerClientBuilder.getInstance().build()
165
+
166
+ private fun stopContainer(containerId: String) {
167
+ if (isRunning(containerId)) {
168
+ dockerClient.stopContainerCmd(containerId).exec()
169
+ }
170
+ }
171
+
172
+ private fun startContainer(containerId: String) {
173
+ if (!isRunning(containerId)) {
174
+ dockerClient.startContainerCmd(containerId).exec()
175
+ }
176
+ }
177
+
178
+ private fun isRunning(containerId: String): Boolean {
179
+ return dockerClient.listContainersCmd().exec().any { container ->
180
+ container.names.any { name ->
181
+ name.contains(containerId)
182
+ }
183
+ }
184
+ }
185
+ }
186
+ }
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-remote
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Shinichi Ishimura
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2017-02-05 00:00:00.000000000 Z
11
+ date: 2017-03-20 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -45,36 +45,39 @@ executables: []
45
45
  extensions: []
46
46
  extra_rdoc_files: []
47
47
  files:
48
+ - .circleci/config.yml
48
49
  - .gitignore
49
- - .travis.yml
50
50
  - Dockerfile
51
51
  - LICENSE.txt
52
52
  - README.md
53
53
  - build.gradle
54
+ - config/checkstyle/checkstyle.xml
55
+ - config/checkstyle/default.xml
54
56
  - docker-compose.yml
57
+ - gradle.properties
55
58
  - gradle/wrapper/gradle-wrapper.jar
56
59
  - gradle/wrapper/gradle-wrapper.properties
57
60
  - gradlew
58
61
  - gradlew.bat
59
62
  - lib/embulk/input/remote.rb
60
- - settings.gradle
61
- - src/main/kotlin/org/embulk/input/RemoteFileInputPlugin.kt
63
+ - src/main/kotlin/org/embulk/input/remote/RemoteFileInputPlugin.kt
62
64
  - src/main/kotlin/org/embulk/input/remote/SSHClient.kt
63
- - src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt
65
+ - src/main/kotlin/org/embulk/input/remote/extensions.kt
66
+ - src/test/kotlin/org/embulk/input/remote/TestRemoteFileInputPlugin.kt
64
67
  - src/test/resources/input/host1/test.csv
65
68
  - src/test/resources/input/host1/test_command.csv
66
69
  - src/test/resources/input/host2/test.csv
67
70
  - src/test/resources/input/host2/test_command.csv
68
71
  - src/test/resources/script/hosts.sh
69
72
  - src/test/resources/yaml/base.yml
70
- - classpath/bcpkix-jdk15on-1.51.jar
73
+ - classpath/embulk-input-remote-0.3.1.jar
74
+ - classpath/sshj-0.19.1.jar
71
75
  - classpath/bcprov-jdk15on-1.51.jar
72
- - classpath/eddsa-0.1.0.jar
73
- - classpath/embulk-input-remote-0.3.0.jar
74
76
  - classpath/jzlib-1.1.3.jar
75
- - classpath/kotlin-runtime-1.0.6.jar
76
- - classpath/kotlin-stdlib-1.0.6.jar
77
- - classpath/sshj-0.19.1.jar
77
+ - classpath/annotations-13.0.jar
78
+ - classpath/bcpkix-jdk15on-1.51.jar
79
+ - classpath/eddsa-0.1.0.jar
80
+ - classpath/kotlin-stdlib-1.1.1.jar
78
81
  homepage: https://github.com/kamatama41/embulk-input-remote
79
82
  licenses:
80
83
  - MIT
data/.travis.yml DELETED
@@ -1,21 +0,0 @@
1
- sudo: required
2
-
3
- language: java
4
-
5
- jdk:
6
- - oraclejdk8
7
-
8
- env:
9
- - >-
10
- KEY_PATH=$PWD/id_rsa_test
11
-
12
- services:
13
- - docker
14
-
15
- before_install:
16
- - ssh-keygen -t ecdsa -f ${KEY_PATH} -N ''
17
- - docker-compose up -d
18
- - docker-compose ps
19
-
20
- script:
21
- - ./gradlew --info check
data/classpath/kotlin-runtime-1.0.6.jar DELETED
Binary file
data/classpath/kotlin-stdlib-1.0.6.jar DELETED
Binary file
data/settings.gradle DELETED
@@ -1,3 +0,0 @@
1
- // Uncomment when using local embulk-test-helpers (and build.gradle as well)
2
- //include ':embulk-test-helpers'
3
- //project(':embulk-test-helpers').projectDir = new File(settingsDir, '../embulk-test-helpers')
data/src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt DELETED
@@ -1,201 +0,0 @@
1
- package org.embulk.input
2
-
3
- import ch.qos.logback.classic.Level
4
- import ch.qos.logback.classic.Logger
5
- import com.github.dockerjava.core.DockerClientBuilder
6
- import org.embulk.config.ConfigSource
7
- import org.embulk.spi.InputPlugin
8
- import org.embulk.test.EmbulkPluginTest
9
- import org.embulk.test.ExtendedEmbulkTests
10
- import org.embulk.test.TestOutputPlugin.assertRecords
11
- import org.embulk.test.TestingEmbulk
12
- import org.embulk.test.Utils.record
13
- import org.hamcrest.CoreMatchers.`is`
14
- import org.hamcrest.MatcherAssert.assertThat
15
- import org.junit.Ignore
16
- import org.junit.Test
17
- import org.slf4j.LoggerFactory
18
- import java.util.Arrays
19
-
20
- class TestRemoteFileInputPlugin : EmbulkPluginTest() {
21
- private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
22
- private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
23
- private val dockerClient = DockerClientBuilder.getInstance().build()
24
-
25
- override fun setup(builder: TestingEmbulk.Builder) {
26
- builder.registerPlugin(InputPlugin::class.java, "remote", RemoteFileInputPlugin::class.java)
27
-
28
- // Setup docker container
29
- startContainer(CONTAINER_ID_HOST1)
30
- startContainer(CONTAINER_ID_HOST2)
31
-
32
- val logLevel = System.getenv("LOG_LEVEL")
33
- if (logLevel != null) {
34
- // Set log level
35
- val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger
36
- rootLogger.level = Level.toLevel(logLevel)
37
- }
38
- }
39
-
40
- @Test fun loadFromRemote() {
41
- runInput(baseConfig())
42
- assertRecords(record(1, "user1"))
43
- }
44
-
45
- @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
46
- @Test fun loadFromRemoteViaPublicKey() {
47
- var keyPath: String? = System.getenv("KEY_PATH")
48
- if (keyPath == null) {
49
- keyPath = "./id_rsa_test"
50
- }
51
-
52
- val publicKeyAuth = newConfig().set("auth", newConfig()
53
- .set("type", "public_key")
54
- .set("key_path", keyPath)
55
- )
56
- runInput(baseConfig().merge(publicKeyAuth))
57
-
58
- assertRecords(record(1, "user1"))
59
- }
60
-
61
- @Test fun testMultiHosts() {
62
- val multiHosts = newConfig()
63
- .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
64
- val config = baseConfig().merge(multiHosts)
65
-
66
- // Run
67
- runInput(config)
68
- assertRecords(
69
- record(1, "user1"),
70
- record(2, "user2")
71
- )
72
- }
73
-
74
- @Test fun loadAllFilesInDirectory() {
75
- val directoryPath = newConfig()
76
- .set("path", "/mount")
77
- val config = baseConfig().merge(directoryPath)
78
-
79
- runInput(config)
80
- assertRecords(
81
- record(1L, "user1"),
82
- record(1L, "command_user1")
83
- )
84
- }
85
-
86
- @Test fun testDefaultPort() {
87
- val defaultPort = newConfig()
88
- .set("hosts", listOf("localhost"))
89
- .set("default_port", 10022)
90
-
91
- runInput(baseConfig().merge(defaultPort))
92
-
93
- assertRecords(record(1L, "user1"))
94
- }
95
-
96
- @Test fun testConfDiff() {
97
- val host2Config = newConfig()
98
- .set("hosts", listOf("localhost:10023"))
99
- var config = baseConfig().merge(host2Config)
100
-
101
- // Run
102
- val runResult = runInput(config)
103
- assertRecords(record(2, "user2"))
104
-
105
- // Re-run with additional host1
106
- val multiHost = newConfig()
107
- .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
108
- config = baseConfig().merge(multiHost)
109
-
110
- runInput(config, runResult.configDiff)
111
-
112
- assertRecords(record(1, "user1"))
113
- }
114
-
115
- @Test fun testResume() {
116
- val multiHost = newConfig()
117
- .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
118
- val config = baseConfig().merge(multiHost)
119
-
120
- // Stop host2 temporarily
121
- stopContainer(CONTAINER_ID_HOST2)
122
-
123
- // Run (but will fail)
124
- var resumableResult = resume(config)
125
-
126
- assertThat(resumableResult.isSuccessful, `is`(false))
127
- assertRecords(record(1, "user1"))
128
-
129
- // Start host2 again
130
- startContainer(CONTAINER_ID_HOST2)
131
-
132
- // Resume
133
- resumableResult = resume(config, resumableResult.resumeState)
134
-
135
- assertThat(resumableResult.isSuccessful, `is`(true))
136
- assertRecords(record(2, "user2"))
137
- }
138
-
139
- @Test fun testIgnoreNotFoundHosts() {
140
- val ignoreNotFoundHosts = newConfig()
141
- .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
142
- .set("ignore_not_found_hosts", true)
143
- val config = baseConfig().merge(ignoreNotFoundHosts)
144
-
145
- // Stop host2
146
- stopContainer(CONTAINER_ID_HOST2)
147
-
148
- // Run (host2 will be ignored)
149
- val resumableResult = resume(config)
150
-
151
- assertThat<Boolean>(resumableResult.isSuccessful, `is`(true))
152
- assertRecords(record(1, "user1"))
153
- }
154
-
155
- @Test fun testCommandOptions() {
156
- val ignoreNotFoundHosts = newConfig()
157
- .set("hosts_command", "./src/test/resources/script/hosts.sh")
158
- .set("hosts_separator", "\n")
159
- .set("path_command", "echo '/mount/test_command.csv'")
160
- val config = baseConfig().merge(ignoreNotFoundHosts)
161
-
162
- runInput(config)
163
-
164
- assertRecords(
165
- record(1, "command_user1"),
166
- record(2, "command_user2")
167
- )
168
- }
169
-
170
- //////////////////////////////
171
- // Helpers
172
- //////////////////////////////
173
-
174
- private fun baseConfig(): ConfigSource {
175
- return ExtendedEmbulkTests.configFromResource("yaml/base.yml")
176
- }
177
-
178
- //////////////////////////////
179
- // Methods for Docker
180
- //////////////////////////////
181
-
182
- private fun stopContainer(containerId: String) {
183
- if (isRunning(containerId)) {
184
- dockerClient.stopContainerCmd(containerId).exec()
185
- }
186
- }
187
-
188
- private fun startContainer(containerId: String) {
189
- if (!isRunning(containerId)) {
190
- dockerClient.startContainerCmd(containerId).exec()
191
- }
192
- }
193
-
194
- private fun isRunning(containerId: String): Boolean {
195
- return dockerClient.listContainersCmd().exec().any { container ->
196
- container.names.any { name ->
197
- name.contains(containerId)
198
- }
199
- }
200
- }
201
- }