embulk-input-remote 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: ea3bfc2c0371ae959be124623397149f674a3228
4
- data.tar.gz: 7b80c2e4233a95201f1481991f7a536ba1bebf9c
3
+ metadata.gz: fe5d5ee6a70519b616bc61b9601d781f14510024
4
+ data.tar.gz: 2bbe3b3615fd7bc8e29be25cb4755294ad8949e2
5
5
  SHA512:
6
- metadata.gz: c635b206f4fdfe0f6e794bada76a2f900ad47f7cd4cbc17f2b6797d6661d5d4f8c279a551c2a20115358b4aa5162683f618504d8a4d7bef30ee7b72a6591ec7c
7
- data.tar.gz: b70b926b21825bc5563634e36a77c9155110fd48f56b5f18f28a84553eb101ac8fb73d4bdf6f279d6144577e2257299e1dc41364bf9ce3cc097acc8f5ddf5f81
6
+ metadata.gz: 0aa4e830ab9deaa103c9487797a9f4ccc4050811fd939f92789b6232c9a082b9014affc75931220d0eb52ced9cc4c4bc8e8c594315d47dc16e85153a38af1ec3
7
+ data.tar.gz: b5a495f9d4f84bdc1dc1490465f8d84d2e8b5e3db308a96f3a909d7b16dc26fb5ad59263be6c74dd94b99f8681497e9614d4f3f844e2602086d810bdfe6c29f0
data/build.gradle CHANGED
@@ -1,9 +1,19 @@
1
+ buildscript {
2
+ ext.kotlin_version = '1.0.6'
3
+ repositories {
4
+ mavenCentral()
5
+ }
6
+ dependencies {
7
+ classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
8
+ }
9
+ }
10
+
1
11
  plugins {
2
12
  id "com.jfrog.bintray" version "1.1"
3
13
  id "com.github.jruby-gradle.base" version "0.1.5"
4
14
  id "java"
5
- id "idea"
6
15
  }
16
+ apply plugin: "kotlin"
7
17
 
8
18
  compileJava {
9
19
  options.compilerArgs = ['-Xlint:all']
@@ -13,25 +23,26 @@ import com.github.jrubygradle.JRubyExec
13
23
  repositories {
14
24
  mavenCentral()
15
25
  jcenter()
26
+ maven { url 'http://kamatama41.github.com/embulk-test-helpers/repository' }
16
27
  }
17
28
  configurations {
18
29
  provided
19
30
  }
20
31
 
21
- version = "0.2.0"
32
+ version = "0.3.0"
33
+ sourceCompatibility = 1.7
34
+ targetCompatibility = 1.7
22
35
 
23
36
  dependencies {
24
- compile "org.embulk:embulk-core:0.8.15"
37
+ compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
38
+ compile "org.embulk:embulk-core:0.8.16"
25
39
  compile "com.hierynomus:sshj:0.19.1"
26
40
  compile "com.jcraft:jzlib:1.1.3"
27
- provided "org.embulk:embulk-core:0.8.15"
28
- testCompile "org.embulk:embulk-standards:0.8.15"
29
- testCompile "org.embulk:embulk-test:0.8.15"
41
+ provided "org.embulk:embulk-core:0.8.16"
42
+ testCompile 'com.kamatama41:embulk-test-helpers:0.1.2'
30
43
  testCompile "com.github.docker-java:docker-java:3.0.7"
31
- // for local testing
32
- // testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
33
- // testCompile 'junit:junit:4.12'
34
- // testCompile 'org.hamcrest:hamcrest-library:1.3'
44
+ // Uncomment when using local embulk-test-helpers (and settings.gradle as well)
45
+ //testCompile project(':embulk-test-helpers')
35
46
  }
36
47
 
37
48
  task classpath(type: Copy, dependsOn: ["jar"]) {
Binary file
Binary file
Binary file
@@ -1,6 +1,6 @@
1
- #Sat Dec 17 23:39:37 JST 2016
1
+ #Thu Feb 02 08:54:57 JST 2017
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip
data/gradlew CHANGED
@@ -1,4 +1,4 @@
1
- #!/usr/bin/env bash
1
+ #!/usr/bin/env sh
2
2
 
3
3
  ##############################################################################
4
4
  ##
@@ -154,11 +154,19 @@ if $cygwin ; then
154
154
  esac
155
155
  fi
156
156
 
157
- # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
158
- function splitJvmOpts() {
159
- JVM_OPTS=("$@")
157
+ # Escape application args
158
+ save ( ) {
159
+ for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
160
+ echo " "
160
161
  }
161
- eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
162
- JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
162
+ APP_ARGS=$(save "$@")
163
163
 
164
- exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
164
+ # Collect all arguments for the java command, following the shell quoting and substitution rules
165
+ eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
166
+
167
+ # by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
168
+ if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
169
+ cd "$(dirname "$0")"
170
+ fi
171
+
172
+ exec "$JAVACMD" "$@"
data/gradlew.bat CHANGED
@@ -49,7 +49,6 @@ goto fail
49
49
  @rem Get command-line arguments, handling Windows variants
50
50
 
51
51
  if not "%OS%" == "Windows_NT" goto win9xME_args
52
- if "%@eval[2+2]" == "4" goto 4NT_args
53
52
 
54
53
  :win9xME_args
55
54
  @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
60
59
  if "x%~1" == "x" goto execute
61
60
 
62
61
  set CMD_LINE_ARGS=%*
63
- goto execute
64
-
65
- :4NT_args
66
- @rem Get arguments from the 4NT Shell from JP Software
67
- set CMD_LINE_ARGS=%$
68
62
 
69
63
  :execute
70
64
  @rem Setup the command line
data/settings.gradle ADDED
@@ -0,0 +1,3 @@
1
+ // Uncomment when using local embulk-test-helpers (and build.gradle as well)
2
+ //include ':embulk-test-helpers'
3
+ //project(':embulk-test-helpers').projectDir = new File(settingsDir, '../embulk-test-helpers')
@@ -0,0 +1,216 @@
1
+ package org.embulk.input
2
+
3
+ import com.fasterxml.jackson.annotation.JsonProperty
4
+ import com.google.common.base.Optional
5
+ import org.embulk.config.Config
6
+ import org.embulk.config.ConfigDefault
7
+ import org.embulk.config.ConfigDiff
8
+ import org.embulk.config.ConfigInject
9
+ import org.embulk.config.ConfigSource
10
+ import org.embulk.config.Task
11
+ import org.embulk.config.TaskReport
12
+ import org.embulk.config.TaskSource
13
+ import org.embulk.input.remote.SSHClient
14
+ import org.embulk.spi.BufferAllocator
15
+ import org.embulk.spi.Exec
16
+ import org.embulk.spi.FileInputPlugin
17
+ import org.embulk.spi.TransactionalFileInput
18
+ import org.embulk.spi.util.InputStreamTransactionalFileInput
19
+ import java.io.BufferedReader
20
+ import java.io.ByteArrayInputStream
21
+ import java.io.ByteArrayOutputStream
22
+ import java.io.IOException
23
+ import java.io.InputStream
24
+ import java.io.InputStreamReader
25
+
26
+
27
+ class RemoteFileInputPlugin : FileInputPlugin {
28
+ interface PluginTask : Task {
29
+ @Config("hosts")
30
+ @ConfigDefault("[]")
31
+ fun getHosts(): List<String>
32
+
33
+ @Config("hosts_command")
34
+ @ConfigDefault("null")
35
+ fun getHostsCommand(): Optional<String>
36
+
37
+ @Config("hosts_separator")
38
+ @ConfigDefault("\" \"")
39
+ fun getHostsSeparator(): String
40
+
41
+ @Config("default_port")
42
+ @ConfigDefault("22")
43
+ fun getDefaultPort(): Int
44
+
45
+ @Config("path")
46
+ @ConfigDefault("\"\"")
47
+ fun getPath(): String
48
+
49
+ @Config("path_command")
50
+ @ConfigDefault("null")
51
+ fun getPathCommand(): Optional<String>
52
+
53
+ @Config("auth")
54
+ fun getAuthConfig(): AuthConfig
55
+
56
+ @Config("ignore_not_found_hosts")
57
+ @ConfigDefault("false")
58
+ fun getIgnoreNotFoundHosts(): Boolean
59
+
60
+ @Config("done_targets")
61
+ @ConfigDefault("[]")
62
+ fun getDoneTargets(): List<Target>
63
+
64
+ fun getTargets(): List<Target>
65
+
66
+ fun setTargets(targets: List<Target>)
67
+
68
+ @ConfigInject
69
+ fun getBufferAllocator(): BufferAllocator
70
+ }
71
+
72
+ interface AuthConfig : Task {
73
+ @Config("type")
74
+ @ConfigDefault("\"public_key\"")
75
+ fun getType(): String
76
+
77
+ @Config("user")
78
+ @ConfigDefault("null")
79
+ fun getUser(): Optional<String>
80
+
81
+ @Config("key_path")
82
+ @ConfigDefault("null")
83
+ fun getKeyPath(): Optional<String>
84
+
85
+ @Config("password")
86
+ @ConfigDefault("null")
87
+ fun getPassword(): Optional<String>
88
+
89
+ @Config("skip_host_key_verification")
90
+ @ConfigDefault("false")
91
+ fun getSkipHostKeyVerification(): Boolean
92
+ }
93
+
94
+ private val log = Exec.getLogger(javaClass)
95
+
96
+ override fun transaction(config: ConfigSource, control: FileInputPlugin.Control): ConfigDiff {
97
+ val task = config.loadConfig(PluginTask::class.java)
98
+ val targets = listTargets(task)
99
+ log.info("Loading targets $targets")
100
+ task.setTargets(targets)
101
+
102
+ // number of processors is same with number of targets
103
+ val taskCount = targets.size
104
+ return resume(task.dump(), taskCount, control)
105
+ }
106
+
107
+ override fun resume(taskSource: TaskSource, taskCount: Int, control: FileInputPlugin.Control): ConfigDiff {
108
+ val task = taskSource.loadTask(PluginTask::class.java)
109
+
110
+ control.run(taskSource, taskCount)
111
+
112
+ return Exec.newConfigDiff().set("done_targets", task.getTargets())
113
+ }
114
+
115
+ override fun cleanup(taskSource: TaskSource, taskCount: Int, successTaskReports: MutableList<TaskReport>) {
116
+ }
117
+
118
+ override fun open(taskSource: TaskSource, taskIndex: Int): TransactionalFileInput {
119
+ val task = taskSource.loadTask(PluginTask::class.java)
120
+ val target = task.getTargets()[taskIndex]
121
+
122
+ return object : InputStreamTransactionalFileInput(task.getBufferAllocator(), { download(target, task) }) {
123
+ override fun abort() {
124
+ }
125
+
126
+ override fun commit(): TaskReport {
127
+ return Exec.newTaskReport()
128
+ }
129
+ }
130
+ }
131
+
132
+ private fun listTargets(task: PluginTask): List<Target> {
133
+ val hosts = listHosts(task)
134
+ val path = getPath(task)
135
+ val doneTargets = task.getDoneTargets()
136
+ val ignoreNotFoundHosts = task.getIgnoreNotFoundHosts()
137
+
138
+ return hosts.map {
139
+ val split = it.split(Regex(":"))
140
+ val host = split[0]
141
+ val port = if (split.size > 1) split[1].toInt() else task.getDefaultPort()
142
+ Target(host, port, path)
143
+ }.filter {
144
+ !doneTargets.contains(it)
145
+ }.filter {
146
+ !ignoreNotFoundHosts || {
147
+ try {
148
+ exists(it, task)
149
+ } catch (e: IOException) {
150
+ log.warn("failed to check the file exists. $it", e)
151
+ false
152
+ }
153
+ }()
154
+ }
155
+ }
156
+
157
+ private fun listHosts(task: PluginTask): List<String> {
158
+ return task.getHostsCommand().transform {
159
+ execCommand(it).split(task.getHostsSeparator().toRegex())
160
+ }.or(task.getHosts())
161
+ }
162
+
163
+ private fun getPath(task: PluginTask): String {
164
+ return task.getPathCommand().transform { execCommand(it) }.or(task.getPath())
165
+ }
166
+
167
+ private fun execCommand(command: String?): String {
168
+ val pb = ProcessBuilder("sh", "-c", command) // TODO: windows
169
+ log.info("Running command $command")
170
+ val process = pb.start()
171
+ process.inputStream.use { stream ->
172
+ BufferedReader(InputStreamReader(stream)).use { brStdout ->
173
+ val stdout = brStdout.readText()
174
+
175
+ val code = process.waitFor()
176
+ if (code != 0) {
177
+ throw IOException("Command finished with non-zero exit code. Exit code is $code")
178
+ }
179
+
180
+ return stdout.trim()
181
+ }
182
+ }
183
+ }
184
+
185
+ private fun exists(target: Target, task: PluginTask): Boolean {
186
+ SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
187
+ val checkCmd = "ls ${target.path}" // TODO: windows
188
+ val timeout = 5/* second */
189
+ val commandResult = client.execCommand(checkCmd, timeout)
190
+
191
+ if (commandResult.status != 0) {
192
+ log.warn("Remote file not found. $target")
193
+ return false
194
+ }
195
+ return true
196
+ }
197
+ }
198
+
199
+ private fun download(target: Target, task: PluginTask): InputStream {
200
+ SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
201
+ val stream = ByteArrayOutputStream()
202
+ client.scpDownload(target.path, stream)
203
+ return ByteArrayInputStream(stream.toByteArray())
204
+ }
205
+ }
206
+
207
+ data class Target constructor(
208
+ @JsonProperty("host") val host: String,
209
+ @JsonProperty("port") val port: Int,
210
+ @JsonProperty("path") val path: String) {
211
+
212
+ override fun toString(): String {
213
+ return "$host:$port:$path"
214
+ }
215
+ }
216
+ }
@@ -0,0 +1,74 @@
1
+ package org.embulk.input.remote
2
+
3
+ import net.schmizz.sshj.DefaultConfig
4
+ import net.schmizz.sshj.transport.verification.PromiscuousVerifier
5
+ import net.schmizz.sshj.xfer.InMemoryDestFile
6
+ import net.schmizz.sshj.xfer.LocalDestFile
7
+ import org.embulk.input.RemoteFileInputPlugin
8
+ import java.io.Closeable
9
+ import java.io.InputStream
10
+ import java.io.OutputStream
11
+ import java.util.concurrent.TimeUnit
12
+
13
+ class SSHClient private constructor(val client: net.schmizz.sshj.SSHClient) : Closeable {
14
+ companion object {
15
+ @JvmStatic
16
+ fun connect(
17
+ host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig
18
+ ): SSHClient {
19
+ val client = SSHClient(net.schmizz.sshj.SSHClient(DefaultConfig()))
20
+ client.connectToHost(host, port, authConfig)
21
+ return client
22
+ }
23
+ }
24
+
25
+ private fun connectToHost(host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig) {
26
+ if (authConfig.getSkipHostKeyVerification()) {
27
+ client.addHostKeyVerifier(PromiscuousVerifier())
28
+ }
29
+ client.loadKnownHosts()
30
+ client.connect(host, port)
31
+
32
+ val type = authConfig.getType()
33
+ val user = authConfig.getUser().or(System.getProperty("user.name"))
34
+
35
+ if ("password" == type) {
36
+ client.authPassword(user, authConfig.getPassword().get())
37
+ } else if ("public_key" == type) {
38
+ authConfig.getKeyPath().transform {
39
+ client.authPublickey(user, it)
40
+ }.or {
41
+ client.authPublickey(user)
42
+ }
43
+ } else {
44
+ throw UnsupportedOperationException("Unsupported auth type : " + type)
45
+ }
46
+ }
47
+
48
+ fun execCommand(command: String, timeoutSecond: Int): CommandResult {
49
+ client.startSession().use { session ->
50
+ val cmd = session.exec(command)
51
+ cmd.join(timeoutSecond.toLong(), TimeUnit.SECONDS)
52
+ return CommandResult(cmd.exitStatus!!, cmd.inputStream)
53
+ }
54
+ }
55
+
56
+ fun scpDownload(path: String, stream: OutputStream) {
57
+ client.useCompression()
58
+ client.newSCPFileTransfer().download(path, object :InMemoryDestFile() {
59
+ override fun getOutputStream(): OutputStream {
60
+ return stream
61
+ }
62
+
63
+ override fun getTargetDirectory(dirname: String): LocalDestFile {
64
+ return this
65
+ }
66
+ })
67
+ }
68
+
69
+ override fun close() {
70
+ client.close()
71
+ }
72
+
73
+ data class CommandResult(val status: Int, val stdout: InputStream)
74
+ }
@@ -0,0 +1,201 @@
1
+ package org.embulk.input
2
+
3
+ import ch.qos.logback.classic.Level
4
+ import ch.qos.logback.classic.Logger
5
+ import com.github.dockerjava.core.DockerClientBuilder
6
+ import org.embulk.config.ConfigSource
7
+ import org.embulk.spi.InputPlugin
8
+ import org.embulk.test.EmbulkPluginTest
9
+ import org.embulk.test.ExtendedEmbulkTests
10
+ import org.embulk.test.TestOutputPlugin.assertRecords
11
+ import org.embulk.test.TestingEmbulk
12
+ import org.embulk.test.Utils.record
13
+ import org.hamcrest.CoreMatchers.`is`
14
+ import org.hamcrest.MatcherAssert.assertThat
15
+ import org.junit.Ignore
16
+ import org.junit.Test
17
+ import org.slf4j.LoggerFactory
18
+ import java.util.Arrays
19
+
20
+ class TestRemoteFileInputPlugin : EmbulkPluginTest() {
21
+ private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
22
+ private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
23
+ private val dockerClient = DockerClientBuilder.getInstance().build()
24
+
25
+ override fun setup(builder: TestingEmbulk.Builder) {
26
+ builder.registerPlugin(InputPlugin::class.java, "remote", RemoteFileInputPlugin::class.java)
27
+
28
+ // Setup docker container
29
+ startContainer(CONTAINER_ID_HOST1)
30
+ startContainer(CONTAINER_ID_HOST2)
31
+
32
+ val logLevel = System.getenv("LOG_LEVEL")
33
+ if (logLevel != null) {
34
+ // Set log level
35
+ val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger
36
+ rootLogger.level = Level.toLevel(logLevel)
37
+ }
38
+ }
39
+
40
+ @Test fun loadFromRemote() {
41
+ runInput(baseConfig())
42
+ assertRecords(record(1, "user1"))
43
+ }
44
+
45
+ @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
46
+ @Test fun loadFromRemoteViaPublicKey() {
47
+ var keyPath: String? = System.getenv("KEY_PATH")
48
+ if (keyPath == null) {
49
+ keyPath = "./id_rsa_test"
50
+ }
51
+
52
+ val publicKeyAuth = newConfig().set("auth", newConfig()
53
+ .set("type", "public_key")
54
+ .set("key_path", keyPath)
55
+ )
56
+ runInput(baseConfig().merge(publicKeyAuth))
57
+
58
+ assertRecords(record(1, "user1"))
59
+ }
60
+
61
+ @Test fun testMultiHosts() {
62
+ val multiHosts = newConfig()
63
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
64
+ val config = baseConfig().merge(multiHosts)
65
+
66
+ // Run
67
+ runInput(config)
68
+ assertRecords(
69
+ record(1, "user1"),
70
+ record(2, "user2")
71
+ )
72
+ }
73
+
74
+ @Test fun loadAllFilesInDirectory() {
75
+ val directoryPath = newConfig()
76
+ .set("path", "/mount")
77
+ val config = baseConfig().merge(directoryPath)
78
+
79
+ runInput(config)
80
+ assertRecords(
81
+ record(1L, "user1"),
82
+ record(1L, "command_user1")
83
+ )
84
+ }
85
+
86
+ @Test fun testDefaultPort() {
87
+ val defaultPort = newConfig()
88
+ .set("hosts", listOf("localhost"))
89
+ .set("default_port", 10022)
90
+
91
+ runInput(baseConfig().merge(defaultPort))
92
+
93
+ assertRecords(record(1L, "user1"))
94
+ }
95
+
96
+ @Test fun testConfDiff() {
97
+ val host2Config = newConfig()
98
+ .set("hosts", listOf("localhost:10023"))
99
+ var config = baseConfig().merge(host2Config)
100
+
101
+ // Run
102
+ val runResult = runInput(config)
103
+ assertRecords(record(2, "user2"))
104
+
105
+ // Re-run with additional host1
106
+ val multiHost = newConfig()
107
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
108
+ config = baseConfig().merge(multiHost)
109
+
110
+ runInput(config, runResult.configDiff)
111
+
112
+ assertRecords(record(1, "user1"))
113
+ }
114
+
115
+ @Test fun testResume() {
116
+ val multiHost = newConfig()
117
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
118
+ val config = baseConfig().merge(multiHost)
119
+
120
+ // Stop host2 temporarily
121
+ stopContainer(CONTAINER_ID_HOST2)
122
+
123
+ // Run (but will fail)
124
+ var resumableResult = resume(config)
125
+
126
+ assertThat(resumableResult.isSuccessful, `is`(false))
127
+ assertRecords(record(1, "user1"))
128
+
129
+ // Start host2 again
130
+ startContainer(CONTAINER_ID_HOST2)
131
+
132
+ // Resume
133
+ resumableResult = resume(config, resumableResult.resumeState)
134
+
135
+ assertThat(resumableResult.isSuccessful, `is`(true))
136
+ assertRecords(record(2, "user2"))
137
+ }
138
+
139
+ @Test fun testIgnoreNotFoundHosts() {
140
+ val ignoreNotFoundHosts = newConfig()
141
+ .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
142
+ .set("ignore_not_found_hosts", true)
143
+ val config = baseConfig().merge(ignoreNotFoundHosts)
144
+
145
+ // Stop host2
146
+ stopContainer(CONTAINER_ID_HOST2)
147
+
148
+ // Run (host2 will be ignored)
149
+ val resumableResult = resume(config)
150
+
151
+ assertThat<Boolean>(resumableResult.isSuccessful, `is`(true))
152
+ assertRecords(record(1, "user1"))
153
+ }
154
+
155
+ @Test fun testCommandOptions() {
156
+ val ignoreNotFoundHosts = newConfig()
157
+ .set("hosts_command", "./src/test/resources/script/hosts.sh")
158
+ .set("hosts_separator", "\n")
159
+ .set("path_command", "echo '/mount/test_command.csv'")
160
+ val config = baseConfig().merge(ignoreNotFoundHosts)
161
+
162
+ runInput(config)
163
+
164
+ assertRecords(
165
+ record(1, "command_user1"),
166
+ record(2, "command_user2")
167
+ )
168
+ }
169
+
170
+ //////////////////////////////
171
+ // Helpers
172
+ //////////////////////////////
173
+
174
+ private fun baseConfig(): ConfigSource {
175
+ return ExtendedEmbulkTests.configFromResource("yaml/base.yml")
176
+ }
177
+
178
+ //////////////////////////////
179
+ // Methods for Docker
180
+ //////////////////////////////
181
+
182
+ private fun stopContainer(containerId: String) {
183
+ if (isRunning(containerId)) {
184
+ dockerClient.stopContainerCmd(containerId).exec()
185
+ }
186
+ }
187
+
188
+ private fun startContainer(containerId: String) {
189
+ if (!isRunning(containerId)) {
190
+ dockerClient.startContainerCmd(containerId).exec()
191
+ }
192
+ }
193
+
194
+ private fun isRunning(containerId: String): Boolean {
195
+ return dockerClient.listContainersCmd().exec().any { container ->
196
+ container.names.any { name ->
197
+ name.contains(containerId)
198
+ }
199
+ }
200
+ }
201
+ }
@@ -0,0 +1,2 @@
1
+ #!/usr/bin/env bash
2
+ echo -e "localhost:10022\nlocalhost:10023\n\n\n\n"