embulk-input-remote 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/build.gradle +21 -10
- data/classpath/embulk-input-remote-0.3.0.jar +0 -0
- data/classpath/kotlin-runtime-1.0.6.jar +0 -0
- data/classpath/kotlin-stdlib-1.0.6.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +15 -7
- data/gradlew.bat +0 -6
- data/settings.gradle +3 -0
- data/src/main/kotlin/org/embulk/input/RemoteFileInputPlugin.kt +216 -0
- data/src/main/kotlin/org/embulk/input/remote/SSHClient.kt +74 -0
- data/src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt +201 -0
- data/src/test/resources/script/hosts.sh +2 -0
- metadata +10 -11
- data/classpath/embulk-input-remote-0.2.0.jar +0 -0
- data/example/csv/sample_01.csv.gz +0 -0
- data/example/example.yml.liquid +0 -29
- data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java +0 -314
- data/src/main/java/org/embulk/input/remote/SSHClient.java +0 -116
- data/src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java +0 -255
- data/src/test/java/org/embulk/test/MemoryOutputPlugin.java +0 -143
- data/src/test/java/org/embulk/test/MyEmbulkTests.java +0 -23
- data/src/test/java/org/embulk/test/MyTestingEmbulk.java +0 -92
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fe5d5ee6a70519b616bc61b9601d781f14510024
|
4
|
+
data.tar.gz: 2bbe3b3615fd7bc8e29be25cb4755294ad8949e2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0aa4e830ab9deaa103c9487797a9f4ccc4050811fd939f92789b6232c9a082b9014affc75931220d0eb52ced9cc4c4bc8e8c594315d47dc16e85153a38af1ec3
|
7
|
+
data.tar.gz: b5a495f9d4f84bdc1dc1490465f8d84d2e8b5e3db308a96f3a909d7b16dc26fb5ad59263be6c74dd94b99f8681497e9614d4f3f844e2602086d810bdfe6c29f0
|
data/build.gradle
CHANGED
@@ -1,9 +1,19 @@
|
|
1
|
+
buildscript {
|
2
|
+
ext.kotlin_version = '1.0.6'
|
3
|
+
repositories {
|
4
|
+
mavenCentral()
|
5
|
+
}
|
6
|
+
dependencies {
|
7
|
+
classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
|
8
|
+
}
|
9
|
+
}
|
10
|
+
|
1
11
|
plugins {
|
2
12
|
id "com.jfrog.bintray" version "1.1"
|
3
13
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
14
|
id "java"
|
5
|
-
id "idea"
|
6
15
|
}
|
16
|
+
apply plugin: "kotlin"
|
7
17
|
|
8
18
|
compileJava {
|
9
19
|
options.compilerArgs = ['-Xlint:all']
|
@@ -13,25 +23,26 @@ import com.github.jrubygradle.JRubyExec
|
|
13
23
|
repositories {
|
14
24
|
mavenCentral()
|
15
25
|
jcenter()
|
26
|
+
maven { url 'http://kamatama41.github.com/embulk-test-helpers/repository' }
|
16
27
|
}
|
17
28
|
configurations {
|
18
29
|
provided
|
19
30
|
}
|
20
31
|
|
21
|
-
version = "0.
|
32
|
+
version = "0.3.0"
|
33
|
+
sourceCompatibility = 1.7
|
34
|
+
targetCompatibility = 1.7
|
22
35
|
|
23
36
|
dependencies {
|
24
|
-
compile
|
37
|
+
compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
|
38
|
+
compile "org.embulk:embulk-core:0.8.16"
|
25
39
|
compile "com.hierynomus:sshj:0.19.1"
|
26
40
|
compile "com.jcraft:jzlib:1.1.3"
|
27
|
-
provided "org.embulk:embulk-core:0.8.
|
28
|
-
testCompile
|
29
|
-
testCompile "org.embulk:embulk-test:0.8.15"
|
41
|
+
provided "org.embulk:embulk-core:0.8.16"
|
42
|
+
testCompile 'com.kamatama41:embulk-test-helpers:0.1.2'
|
30
43
|
testCompile "com.github.docker-java:docker-java:3.0.7"
|
31
|
-
//
|
32
|
-
//
|
33
|
-
// testCompile 'junit:junit:4.12'
|
34
|
-
// testCompile 'org.hamcrest:hamcrest-library:1.3'
|
44
|
+
// Uncomment when using local embulk-test-helpers (and settings.gradle as well)
|
45
|
+
//testCompile project(':embulk-test-helpers')
|
35
46
|
}
|
36
47
|
|
37
48
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
Binary file
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Thu Feb 02 08:54:57 JST 2017
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
|
data/gradlew
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
#!/usr/bin/env
|
1
|
+
#!/usr/bin/env sh
|
2
2
|
|
3
3
|
##############################################################################
|
4
4
|
##
|
@@ -154,11 +154,19 @@ if $cygwin ; then
|
|
154
154
|
esac
|
155
155
|
fi
|
156
156
|
|
157
|
-
#
|
158
|
-
|
159
|
-
|
157
|
+
# Escape application args
|
158
|
+
save ( ) {
|
159
|
+
for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
|
160
|
+
echo " "
|
160
161
|
}
|
161
|
-
|
162
|
-
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
|
162
|
+
APP_ARGS=$(save "$@")
|
163
163
|
|
164
|
-
|
164
|
+
# Collect all arguments for the java command, following the shell quoting and substitution rules
|
165
|
+
eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
|
166
|
+
|
167
|
+
# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
|
168
|
+
if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
|
169
|
+
cd "$(dirname "$0")"
|
170
|
+
fi
|
171
|
+
|
172
|
+
exec "$JAVACMD" "$@"
|
data/gradlew.bat
CHANGED
@@ -49,7 +49,6 @@ goto fail
|
|
49
49
|
@rem Get command-line arguments, handling Windows variants
|
50
50
|
|
51
51
|
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
52
|
|
54
53
|
:win9xME_args
|
55
54
|
@rem Slurp the command line arguments.
|
@@ -60,11 +59,6 @@ set _SKIP=2
|
|
60
59
|
if "x%~1" == "x" goto execute
|
61
60
|
|
62
61
|
set CMD_LINE_ARGS=%*
|
63
|
-
goto execute
|
64
|
-
|
65
|
-
:4NT_args
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
-
set CMD_LINE_ARGS=%$
|
68
62
|
|
69
63
|
:execute
|
70
64
|
@rem Setup the command line
|
data/settings.gradle
ADDED
@@ -0,0 +1,216 @@
|
|
1
|
+
package org.embulk.input
|
2
|
+
|
3
|
+
import com.fasterxml.jackson.annotation.JsonProperty
|
4
|
+
import com.google.common.base.Optional
|
5
|
+
import org.embulk.config.Config
|
6
|
+
import org.embulk.config.ConfigDefault
|
7
|
+
import org.embulk.config.ConfigDiff
|
8
|
+
import org.embulk.config.ConfigInject
|
9
|
+
import org.embulk.config.ConfigSource
|
10
|
+
import org.embulk.config.Task
|
11
|
+
import org.embulk.config.TaskReport
|
12
|
+
import org.embulk.config.TaskSource
|
13
|
+
import org.embulk.input.remote.SSHClient
|
14
|
+
import org.embulk.spi.BufferAllocator
|
15
|
+
import org.embulk.spi.Exec
|
16
|
+
import org.embulk.spi.FileInputPlugin
|
17
|
+
import org.embulk.spi.TransactionalFileInput
|
18
|
+
import org.embulk.spi.util.InputStreamTransactionalFileInput
|
19
|
+
import java.io.BufferedReader
|
20
|
+
import java.io.ByteArrayInputStream
|
21
|
+
import java.io.ByteArrayOutputStream
|
22
|
+
import java.io.IOException
|
23
|
+
import java.io.InputStream
|
24
|
+
import java.io.InputStreamReader
|
25
|
+
|
26
|
+
|
27
|
+
class RemoteFileInputPlugin : FileInputPlugin {
|
28
|
+
interface PluginTask : Task {
|
29
|
+
@Config("hosts")
|
30
|
+
@ConfigDefault("[]")
|
31
|
+
fun getHosts(): List<String>
|
32
|
+
|
33
|
+
@Config("hosts_command")
|
34
|
+
@ConfigDefault("null")
|
35
|
+
fun getHostsCommand(): Optional<String>
|
36
|
+
|
37
|
+
@Config("hosts_separator")
|
38
|
+
@ConfigDefault("\" \"")
|
39
|
+
fun getHostsSeparator(): String
|
40
|
+
|
41
|
+
@Config("default_port")
|
42
|
+
@ConfigDefault("22")
|
43
|
+
fun getDefaultPort(): Int
|
44
|
+
|
45
|
+
@Config("path")
|
46
|
+
@ConfigDefault("\"\"")
|
47
|
+
fun getPath(): String
|
48
|
+
|
49
|
+
@Config("path_command")
|
50
|
+
@ConfigDefault("null")
|
51
|
+
fun getPathCommand(): Optional<String>
|
52
|
+
|
53
|
+
@Config("auth")
|
54
|
+
fun getAuthConfig(): AuthConfig
|
55
|
+
|
56
|
+
@Config("ignore_not_found_hosts")
|
57
|
+
@ConfigDefault("false")
|
58
|
+
fun getIgnoreNotFoundHosts(): Boolean
|
59
|
+
|
60
|
+
@Config("done_targets")
|
61
|
+
@ConfigDefault("[]")
|
62
|
+
fun getDoneTargets(): List<Target>
|
63
|
+
|
64
|
+
fun getTargets(): List<Target>
|
65
|
+
|
66
|
+
fun setTargets(targets: List<Target>)
|
67
|
+
|
68
|
+
@ConfigInject
|
69
|
+
fun getBufferAllocator(): BufferAllocator
|
70
|
+
}
|
71
|
+
|
72
|
+
interface AuthConfig : Task {
|
73
|
+
@Config("type")
|
74
|
+
@ConfigDefault("\"public_key\"")
|
75
|
+
fun getType(): String
|
76
|
+
|
77
|
+
@Config("user")
|
78
|
+
@ConfigDefault("null")
|
79
|
+
fun getUser(): Optional<String>
|
80
|
+
|
81
|
+
@Config("key_path")
|
82
|
+
@ConfigDefault("null")
|
83
|
+
fun getKeyPath(): Optional<String>
|
84
|
+
|
85
|
+
@Config("password")
|
86
|
+
@ConfigDefault("null")
|
87
|
+
fun getPassword(): Optional<String>
|
88
|
+
|
89
|
+
@Config("skip_host_key_verification")
|
90
|
+
@ConfigDefault("false")
|
91
|
+
fun getSkipHostKeyVerification(): Boolean
|
92
|
+
}
|
93
|
+
|
94
|
+
private val log = Exec.getLogger(javaClass)
|
95
|
+
|
96
|
+
override fun transaction(config: ConfigSource, control: FileInputPlugin.Control): ConfigDiff {
|
97
|
+
val task = config.loadConfig(PluginTask::class.java)
|
98
|
+
val targets = listTargets(task)
|
99
|
+
log.info("Loading targets $targets")
|
100
|
+
task.setTargets(targets)
|
101
|
+
|
102
|
+
// number of processors is same with number of targets
|
103
|
+
val taskCount = targets.size
|
104
|
+
return resume(task.dump(), taskCount, control)
|
105
|
+
}
|
106
|
+
|
107
|
+
override fun resume(taskSource: TaskSource, taskCount: Int, control: FileInputPlugin.Control): ConfigDiff {
|
108
|
+
val task = taskSource.loadTask(PluginTask::class.java)
|
109
|
+
|
110
|
+
control.run(taskSource, taskCount)
|
111
|
+
|
112
|
+
return Exec.newConfigDiff().set("done_targets", task.getTargets())
|
113
|
+
}
|
114
|
+
|
115
|
+
override fun cleanup(taskSource: TaskSource, taskCount: Int, successTaskReports: MutableList<TaskReport>) {
|
116
|
+
}
|
117
|
+
|
118
|
+
override fun open(taskSource: TaskSource, taskIndex: Int): TransactionalFileInput {
|
119
|
+
val task = taskSource.loadTask(PluginTask::class.java)
|
120
|
+
val target = task.getTargets()[taskIndex]
|
121
|
+
|
122
|
+
return object : InputStreamTransactionalFileInput(task.getBufferAllocator(), { download(target, task) }) {
|
123
|
+
override fun abort() {
|
124
|
+
}
|
125
|
+
|
126
|
+
override fun commit(): TaskReport {
|
127
|
+
return Exec.newTaskReport()
|
128
|
+
}
|
129
|
+
}
|
130
|
+
}
|
131
|
+
|
132
|
+
private fun listTargets(task: PluginTask): List<Target> {
|
133
|
+
val hosts = listHosts(task)
|
134
|
+
val path = getPath(task)
|
135
|
+
val doneTargets = task.getDoneTargets()
|
136
|
+
val ignoreNotFoundHosts = task.getIgnoreNotFoundHosts()
|
137
|
+
|
138
|
+
return hosts.map {
|
139
|
+
val split = it.split(Regex(":"))
|
140
|
+
val host = split[0]
|
141
|
+
val port = if (split.size > 1) split[1].toInt() else task.getDefaultPort()
|
142
|
+
Target(host, port, path)
|
143
|
+
}.filter {
|
144
|
+
!doneTargets.contains(it)
|
145
|
+
}.filter {
|
146
|
+
!ignoreNotFoundHosts || {
|
147
|
+
try {
|
148
|
+
exists(it, task)
|
149
|
+
} catch (e: IOException) {
|
150
|
+
log.warn("failed to check the file exists. $it", e)
|
151
|
+
false
|
152
|
+
}
|
153
|
+
}()
|
154
|
+
}
|
155
|
+
}
|
156
|
+
|
157
|
+
private fun listHosts(task: PluginTask): List<String> {
|
158
|
+
return task.getHostsCommand().transform {
|
159
|
+
execCommand(it).split(task.getHostsSeparator().toRegex())
|
160
|
+
}.or(task.getHosts())
|
161
|
+
}
|
162
|
+
|
163
|
+
private fun getPath(task: PluginTask): String {
|
164
|
+
return task.getPathCommand().transform { execCommand(it) }.or(task.getPath())
|
165
|
+
}
|
166
|
+
|
167
|
+
private fun execCommand(command: String?): String {
|
168
|
+
val pb = ProcessBuilder("sh", "-c", command) // TODO: windows
|
169
|
+
log.info("Running command $command")
|
170
|
+
val process = pb.start()
|
171
|
+
process.inputStream.use { stream ->
|
172
|
+
BufferedReader(InputStreamReader(stream)).use { brStdout ->
|
173
|
+
val stdout = brStdout.readText()
|
174
|
+
|
175
|
+
val code = process.waitFor()
|
176
|
+
if (code != 0) {
|
177
|
+
throw IOException("Command finished with non-zero exit code. Exit code is $code")
|
178
|
+
}
|
179
|
+
|
180
|
+
return stdout.trim()
|
181
|
+
}
|
182
|
+
}
|
183
|
+
}
|
184
|
+
|
185
|
+
private fun exists(target: Target, task: PluginTask): Boolean {
|
186
|
+
SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
|
187
|
+
val checkCmd = "ls ${target.path}" // TODO: windows
|
188
|
+
val timeout = 5/* second */
|
189
|
+
val commandResult = client.execCommand(checkCmd, timeout)
|
190
|
+
|
191
|
+
if (commandResult.status != 0) {
|
192
|
+
log.warn("Remote file not found. $target")
|
193
|
+
return false
|
194
|
+
}
|
195
|
+
return true
|
196
|
+
}
|
197
|
+
}
|
198
|
+
|
199
|
+
private fun download(target: Target, task: PluginTask): InputStream {
|
200
|
+
SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
|
201
|
+
val stream = ByteArrayOutputStream()
|
202
|
+
client.scpDownload(target.path, stream)
|
203
|
+
return ByteArrayInputStream(stream.toByteArray())
|
204
|
+
}
|
205
|
+
}
|
206
|
+
|
207
|
+
data class Target constructor(
|
208
|
+
@JsonProperty("host") val host: String,
|
209
|
+
@JsonProperty("port") val port: Int,
|
210
|
+
@JsonProperty("path") val path: String) {
|
211
|
+
|
212
|
+
override fun toString(): String {
|
213
|
+
return "$host:$port:$path"
|
214
|
+
}
|
215
|
+
}
|
216
|
+
}
|
@@ -0,0 +1,74 @@
|
|
1
|
+
package org.embulk.input.remote
|
2
|
+
|
3
|
+
import net.schmizz.sshj.DefaultConfig
|
4
|
+
import net.schmizz.sshj.transport.verification.PromiscuousVerifier
|
5
|
+
import net.schmizz.sshj.xfer.InMemoryDestFile
|
6
|
+
import net.schmizz.sshj.xfer.LocalDestFile
|
7
|
+
import org.embulk.input.RemoteFileInputPlugin
|
8
|
+
import java.io.Closeable
|
9
|
+
import java.io.InputStream
|
10
|
+
import java.io.OutputStream
|
11
|
+
import java.util.concurrent.TimeUnit
|
12
|
+
|
13
|
+
class SSHClient private constructor(val client: net.schmizz.sshj.SSHClient) : Closeable {
|
14
|
+
companion object {
|
15
|
+
@JvmStatic
|
16
|
+
fun connect(
|
17
|
+
host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig
|
18
|
+
): SSHClient {
|
19
|
+
val client = SSHClient(net.schmizz.sshj.SSHClient(DefaultConfig()))
|
20
|
+
client.connectToHost(host, port, authConfig)
|
21
|
+
return client
|
22
|
+
}
|
23
|
+
}
|
24
|
+
|
25
|
+
private fun connectToHost(host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig) {
|
26
|
+
if (authConfig.getSkipHostKeyVerification()) {
|
27
|
+
client.addHostKeyVerifier(PromiscuousVerifier())
|
28
|
+
}
|
29
|
+
client.loadKnownHosts()
|
30
|
+
client.connect(host, port)
|
31
|
+
|
32
|
+
val type = authConfig.getType()
|
33
|
+
val user = authConfig.getUser().or(System.getProperty("user.name"))
|
34
|
+
|
35
|
+
if ("password" == type) {
|
36
|
+
client.authPassword(user, authConfig.getPassword().get())
|
37
|
+
} else if ("public_key" == type) {
|
38
|
+
authConfig.getKeyPath().transform {
|
39
|
+
client.authPublickey(user, it)
|
40
|
+
}.or {
|
41
|
+
client.authPublickey(user)
|
42
|
+
}
|
43
|
+
} else {
|
44
|
+
throw UnsupportedOperationException("Unsupported auth type : " + type)
|
45
|
+
}
|
46
|
+
}
|
47
|
+
|
48
|
+
fun execCommand(command: String, timeoutSecond: Int): CommandResult {
|
49
|
+
client.startSession().use { session ->
|
50
|
+
val cmd = session.exec(command)
|
51
|
+
cmd.join(timeoutSecond.toLong(), TimeUnit.SECONDS)
|
52
|
+
return CommandResult(cmd.exitStatus!!, cmd.inputStream)
|
53
|
+
}
|
54
|
+
}
|
55
|
+
|
56
|
+
fun scpDownload(path: String, stream: OutputStream) {
|
57
|
+
client.useCompression()
|
58
|
+
client.newSCPFileTransfer().download(path, object :InMemoryDestFile() {
|
59
|
+
override fun getOutputStream(): OutputStream {
|
60
|
+
return stream
|
61
|
+
}
|
62
|
+
|
63
|
+
override fun getTargetDirectory(dirname: String): LocalDestFile {
|
64
|
+
return this
|
65
|
+
}
|
66
|
+
})
|
67
|
+
}
|
68
|
+
|
69
|
+
override fun close() {
|
70
|
+
client.close()
|
71
|
+
}
|
72
|
+
|
73
|
+
data class CommandResult(val status: Int, val stdout: InputStream)
|
74
|
+
}
|
@@ -0,0 +1,201 @@
|
|
1
|
+
package org.embulk.input
|
2
|
+
|
3
|
+
import ch.qos.logback.classic.Level
|
4
|
+
import ch.qos.logback.classic.Logger
|
5
|
+
import com.github.dockerjava.core.DockerClientBuilder
|
6
|
+
import org.embulk.config.ConfigSource
|
7
|
+
import org.embulk.spi.InputPlugin
|
8
|
+
import org.embulk.test.EmbulkPluginTest
|
9
|
+
import org.embulk.test.ExtendedEmbulkTests
|
10
|
+
import org.embulk.test.TestOutputPlugin.assertRecords
|
11
|
+
import org.embulk.test.TestingEmbulk
|
12
|
+
import org.embulk.test.Utils.record
|
13
|
+
import org.hamcrest.CoreMatchers.`is`
|
14
|
+
import org.hamcrest.MatcherAssert.assertThat
|
15
|
+
import org.junit.Ignore
|
16
|
+
import org.junit.Test
|
17
|
+
import org.slf4j.LoggerFactory
|
18
|
+
import java.util.Arrays
|
19
|
+
|
20
|
+
class TestRemoteFileInputPlugin : EmbulkPluginTest() {
|
21
|
+
private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
|
22
|
+
private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
|
23
|
+
private val dockerClient = DockerClientBuilder.getInstance().build()
|
24
|
+
|
25
|
+
override fun setup(builder: TestingEmbulk.Builder) {
|
26
|
+
builder.registerPlugin(InputPlugin::class.java, "remote", RemoteFileInputPlugin::class.java)
|
27
|
+
|
28
|
+
// Setup docker container
|
29
|
+
startContainer(CONTAINER_ID_HOST1)
|
30
|
+
startContainer(CONTAINER_ID_HOST2)
|
31
|
+
|
32
|
+
val logLevel = System.getenv("LOG_LEVEL")
|
33
|
+
if (logLevel != null) {
|
34
|
+
// Set log level
|
35
|
+
val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger
|
36
|
+
rootLogger.level = Level.toLevel(logLevel)
|
37
|
+
}
|
38
|
+
}
|
39
|
+
|
40
|
+
@Test fun loadFromRemote() {
|
41
|
+
runInput(baseConfig())
|
42
|
+
assertRecords(record(1, "user1"))
|
43
|
+
}
|
44
|
+
|
45
|
+
@Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
|
46
|
+
@Test fun loadFromRemoteViaPublicKey() {
|
47
|
+
var keyPath: String? = System.getenv("KEY_PATH")
|
48
|
+
if (keyPath == null) {
|
49
|
+
keyPath = "./id_rsa_test"
|
50
|
+
}
|
51
|
+
|
52
|
+
val publicKeyAuth = newConfig().set("auth", newConfig()
|
53
|
+
.set("type", "public_key")
|
54
|
+
.set("key_path", keyPath)
|
55
|
+
)
|
56
|
+
runInput(baseConfig().merge(publicKeyAuth))
|
57
|
+
|
58
|
+
assertRecords(record(1, "user1"))
|
59
|
+
}
|
60
|
+
|
61
|
+
@Test fun testMultiHosts() {
|
62
|
+
val multiHosts = newConfig()
|
63
|
+
.set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
|
64
|
+
val config = baseConfig().merge(multiHosts)
|
65
|
+
|
66
|
+
// Run
|
67
|
+
runInput(config)
|
68
|
+
assertRecords(
|
69
|
+
record(1, "user1"),
|
70
|
+
record(2, "user2")
|
71
|
+
)
|
72
|
+
}
|
73
|
+
|
74
|
+
@Test fun loadAllFilesInDirectory() {
|
75
|
+
val directoryPath = newConfig()
|
76
|
+
.set("path", "/mount")
|
77
|
+
val config = baseConfig().merge(directoryPath)
|
78
|
+
|
79
|
+
runInput(config)
|
80
|
+
assertRecords(
|
81
|
+
record(1L, "user1"),
|
82
|
+
record(1L, "command_user1")
|
83
|
+
)
|
84
|
+
}
|
85
|
+
|
86
|
+
@Test fun testDefaultPort() {
|
87
|
+
val defaultPort = newConfig()
|
88
|
+
.set("hosts", listOf("localhost"))
|
89
|
+
.set("default_port", 10022)
|
90
|
+
|
91
|
+
runInput(baseConfig().merge(defaultPort))
|
92
|
+
|
93
|
+
assertRecords(record(1L, "user1"))
|
94
|
+
}
|
95
|
+
|
96
|
+
@Test fun testConfDiff() {
|
97
|
+
val host2Config = newConfig()
|
98
|
+
.set("hosts", listOf("localhost:10023"))
|
99
|
+
var config = baseConfig().merge(host2Config)
|
100
|
+
|
101
|
+
// Run
|
102
|
+
val runResult = runInput(config)
|
103
|
+
assertRecords(record(2, "user2"))
|
104
|
+
|
105
|
+
// Re-run with additional host1
|
106
|
+
val multiHost = newConfig()
|
107
|
+
.set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
|
108
|
+
config = baseConfig().merge(multiHost)
|
109
|
+
|
110
|
+
runInput(config, runResult.configDiff)
|
111
|
+
|
112
|
+
assertRecords(record(1, "user1"))
|
113
|
+
}
|
114
|
+
|
115
|
+
@Test fun testResume() {
|
116
|
+
val multiHost = newConfig()
|
117
|
+
.set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
|
118
|
+
val config = baseConfig().merge(multiHost)
|
119
|
+
|
120
|
+
// Stop host2 temporarily
|
121
|
+
stopContainer(CONTAINER_ID_HOST2)
|
122
|
+
|
123
|
+
// Run (but will fail)
|
124
|
+
var resumableResult = resume(config)
|
125
|
+
|
126
|
+
assertThat(resumableResult.isSuccessful, `is`(false))
|
127
|
+
assertRecords(record(1, "user1"))
|
128
|
+
|
129
|
+
// Start host2 again
|
130
|
+
startContainer(CONTAINER_ID_HOST2)
|
131
|
+
|
132
|
+
// Resume
|
133
|
+
resumableResult = resume(config, resumableResult.resumeState)
|
134
|
+
|
135
|
+
assertThat(resumableResult.isSuccessful, `is`(true))
|
136
|
+
assertRecords(record(2, "user2"))
|
137
|
+
}
|
138
|
+
|
139
|
+
@Test fun testIgnoreNotFoundHosts() {
|
140
|
+
val ignoreNotFoundHosts = newConfig()
|
141
|
+
.set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
|
142
|
+
.set("ignore_not_found_hosts", true)
|
143
|
+
val config = baseConfig().merge(ignoreNotFoundHosts)
|
144
|
+
|
145
|
+
// Stop host2
|
146
|
+
stopContainer(CONTAINER_ID_HOST2)
|
147
|
+
|
148
|
+
// Run (host2 will be ignored)
|
149
|
+
val resumableResult = resume(config)
|
150
|
+
|
151
|
+
assertThat<Boolean>(resumableResult.isSuccessful, `is`(true))
|
152
|
+
assertRecords(record(1, "user1"))
|
153
|
+
}
|
154
|
+
|
155
|
+
@Test fun testCommandOptions() {
|
156
|
+
val ignoreNotFoundHosts = newConfig()
|
157
|
+
.set("hosts_command", "./src/test/resources/script/hosts.sh")
|
158
|
+
.set("hosts_separator", "\n")
|
159
|
+
.set("path_command", "echo '/mount/test_command.csv'")
|
160
|
+
val config = baseConfig().merge(ignoreNotFoundHosts)
|
161
|
+
|
162
|
+
runInput(config)
|
163
|
+
|
164
|
+
assertRecords(
|
165
|
+
record(1, "command_user1"),
|
166
|
+
record(2, "command_user2")
|
167
|
+
)
|
168
|
+
}
|
169
|
+
|
170
|
+
//////////////////////////////
|
171
|
+
// Helpers
|
172
|
+
//////////////////////////////
|
173
|
+
|
174
|
+
private fun baseConfig(): ConfigSource {
|
175
|
+
return ExtendedEmbulkTests.configFromResource("yaml/base.yml")
|
176
|
+
}
|
177
|
+
|
178
|
+
//////////////////////////////
|
179
|
+
// Methods for Docker
|
180
|
+
//////////////////////////////
|
181
|
+
|
182
|
+
private fun stopContainer(containerId: String) {
|
183
|
+
if (isRunning(containerId)) {
|
184
|
+
dockerClient.stopContainerCmd(containerId).exec()
|
185
|
+
}
|
186
|
+
}
|
187
|
+
|
188
|
+
private fun startContainer(containerId: String) {
|
189
|
+
if (!isRunning(containerId)) {
|
190
|
+
dockerClient.startContainerCmd(containerId).exec()
|
191
|
+
}
|
192
|
+
}
|
193
|
+
|
194
|
+
private fun isRunning(containerId: String): Boolean {
|
195
|
+
return dockerClient.listContainersCmd().exec().any { container ->
|
196
|
+
container.names.any { name ->
|
197
|
+
name.contains(containerId)
|
198
|
+
}
|
199
|
+
}
|
200
|
+
}
|
201
|
+
}
|