embulk-input-remote 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea3bfc2c0371ae959be124623397149f674a3228
-  data.tar.gz: 7b80c2e4233a95201f1481991f7a536ba1bebf9c
+  metadata.gz: fe5d5ee6a70519b616bc61b9601d781f14510024
+  data.tar.gz: 2bbe3b3615fd7bc8e29be25cb4755294ad8949e2
 SHA512:
-  metadata.gz: c635b206f4fdfe0f6e794bada76a2f900ad47f7cd4cbc17f2b6797d6661d5d4f8c279a551c2a20115358b4aa5162683f618504d8a4d7bef30ee7b72a6591ec7c
-  data.tar.gz: b70b926b21825bc5563634e36a77c9155110fd48f56b5f18f28a84553eb101ac8fb73d4bdf6f279d6144577e2257299e1dc41364bf9ce3cc097acc8f5ddf5f81
+  metadata.gz: 0aa4e830ab9deaa103c9487797a9f4ccc4050811fd939f92789b6232c9a082b9014affc75931220d0eb52ced9cc4c4bc8e8c594315d47dc16e85153a38af1ec3
+  data.tar.gz: b5a495f9d4f84bdc1dc1490465f8d84d2e8b5e3db308a96f3a909d7b16dc26fb5ad59263be6c74dd94b99f8681497e9614d4f3f844e2602086d810bdfe6c29f0

data/build.gradle

@@ -1,9 +1,19 @@
+buildscript {
+    ext.kotlin_version = '1.0.6'
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
+    }
+}
+
 plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
-    id "idea"
 }
+apply plugin: "kotlin"
 
 compileJava {
     options.compilerArgs = ['-Xlint:all']
@@ -13,25 +23,26 @@ import com.github.jrubygradle.JRubyExec
 repositories {
     mavenCentral()
     jcenter()
+    maven { url 'http://kamatama41.github.com/embulk-test-helpers/repository' }
 }
 configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.3.0"
+sourceCompatibility = 1.7
+targetCompatibility = 1.7
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.8.15"
+    compile "org.jetbrains.kotlin:kotlin-stdlib:$kotlin_version"
+    compile  "org.embulk:embulk-core:0.8.16"
     compile  "com.hierynomus:sshj:0.19.1"
     compile  "com.jcraft:jzlib:1.1.3"
-    provided "org.embulk:embulk-core:0.8.15"
-    testCompile "org.embulk:embulk-standards:0.8.15"
-    testCompile "org.embulk:embulk-test:0.8.15"
+    provided "org.embulk:embulk-core:0.8.16"
+    testCompile 'com.kamatama41:embulk-test-helpers:0.1.2'
     testCompile "com.github.docker-java:docker-java:3.0.7"
-// for local testing
-//    testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
-//    testCompile 'junit:junit:4.12'
-//    testCompile 'org.hamcrest:hamcrest-library:1.3'
+    // Uncomment when using local embulk-test-helpers (and settings.gradle as well)
+    //testCompile project(':embulk-test-helpers')
 }
 
 task classpath(type: Copy, dependsOn: ["jar"]) {

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Sat Dec 17 23:39:37 JST 2016
+#Thu Feb 02 08:54:57 JST 2017
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip

data/gradlew

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/usr/bin/env sh
 
 ##############################################################################
 ##
@@ -154,11 +154,19 @@ if $cygwin ; then
     esac
 fi
 
-# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
-function splitJvmOpts() {
-    JVM_OPTS=("$@")
+# Escape application args
+save ( ) {
+    for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done
+    echo " "
 }
-eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
-JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
+APP_ARGS=$(save "$@")
 
-exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"
+# Collect all arguments for the java command, following the shell quoting and substitution rules
+eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS"
+
+# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong
+if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then
+  cd "$(dirname "$0")"
+fi
+
+exec "$JAVACMD" "$@"

data/gradlew.bat

@@ -49,7 +49,6 @@ goto fail
 @rem Get command-line arguments, handling Windows variants
 
 if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
 
 :win9xME_args
 @rem Slurp the command line arguments.
@@ -60,11 +59,6 @@ set _SKIP=2
 if "x%~1" == "x" goto execute
 
 set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
 
 :execute
 @rem Setup the command line

data/settings.gradle

@@ -0,0 +1,3 @@
+// Uncomment when using local embulk-test-helpers (and build.gradle as well)
+//include ':embulk-test-helpers'
+//project(':embulk-test-helpers').projectDir = new File(settingsDir, '../embulk-test-helpers')

data/src/main/kotlin/org/embulk/input/RemoteFileInputPlugin.kt

@@ -0,0 +1,216 @@
+package org.embulk.input
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import com.google.common.base.Optional
+import org.embulk.config.Config
+import org.embulk.config.ConfigDefault
+import org.embulk.config.ConfigDiff
+import org.embulk.config.ConfigInject
+import org.embulk.config.ConfigSource
+import org.embulk.config.Task
+import org.embulk.config.TaskReport
+import org.embulk.config.TaskSource
+import org.embulk.input.remote.SSHClient
+import org.embulk.spi.BufferAllocator
+import org.embulk.spi.Exec
+import org.embulk.spi.FileInputPlugin
+import org.embulk.spi.TransactionalFileInput
+import org.embulk.spi.util.InputStreamTransactionalFileInput
+import java.io.BufferedReader
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.io.IOException
+import java.io.InputStream
+import java.io.InputStreamReader
+
+
+class RemoteFileInputPlugin : FileInputPlugin {
+    interface PluginTask : Task {
+        @Config("hosts")
+        @ConfigDefault("[]")
+        fun getHosts(): List<String>
+
+        @Config("hosts_command")
+        @ConfigDefault("null")
+        fun getHostsCommand(): Optional<String>
+
+        @Config("hosts_separator")
+        @ConfigDefault("\" \"")
+        fun getHostsSeparator(): String
+
+        @Config("default_port")
+        @ConfigDefault("22")
+        fun getDefaultPort(): Int
+
+        @Config("path")
+        @ConfigDefault("\"\"")
+        fun getPath(): String
+
+        @Config("path_command")
+        @ConfigDefault("null")
+        fun getPathCommand(): Optional<String>
+
+        @Config("auth")
+        fun getAuthConfig(): AuthConfig
+
+        @Config("ignore_not_found_hosts")
+        @ConfigDefault("false")
+        fun getIgnoreNotFoundHosts(): Boolean
+
+        @Config("done_targets")
+        @ConfigDefault("[]")
+        fun getDoneTargets(): List<Target>
+
+        fun getTargets(): List<Target>
+
+        fun setTargets(targets: List<Target>)
+
+        @ConfigInject
+        fun getBufferAllocator(): BufferAllocator
+    }
+
+    interface AuthConfig : Task {
+        @Config("type")
+        @ConfigDefault("\"public_key\"")
+        fun getType(): String
+
+        @Config("user")
+        @ConfigDefault("null")
+        fun getUser(): Optional<String>
+
+        @Config("key_path")
+        @ConfigDefault("null")
+        fun getKeyPath(): Optional<String>
+
+        @Config("password")
+        @ConfigDefault("null")
+        fun getPassword(): Optional<String>
+
+        @Config("skip_host_key_verification")
+        @ConfigDefault("false")
+        fun getSkipHostKeyVerification(): Boolean
+    }
+
+    private val log = Exec.getLogger(javaClass)
+
+    override fun transaction(config: ConfigSource, control: FileInputPlugin.Control): ConfigDiff {
+        val task = config.loadConfig(PluginTask::class.java)
+        val targets = listTargets(task)
+        log.info("Loading targets $targets")
+        task.setTargets(targets)
+
+        // number of processors is same with number of targets
+        val taskCount = targets.size
+        return resume(task.dump(), taskCount, control)
+    }
+
+    override fun resume(taskSource: TaskSource, taskCount: Int, control: FileInputPlugin.Control): ConfigDiff {
+        val task = taskSource.loadTask(PluginTask::class.java)
+
+        control.run(taskSource, taskCount)
+
+        return Exec.newConfigDiff().set("done_targets", task.getTargets())
+    }
+
+    override fun cleanup(taskSource: TaskSource, taskCount: Int, successTaskReports: MutableList<TaskReport>) {
+    }
+
+    override fun open(taskSource: TaskSource, taskIndex: Int): TransactionalFileInput {
+        val task = taskSource.loadTask(PluginTask::class.java)
+        val target = task.getTargets()[taskIndex]
+
+        return object : InputStreamTransactionalFileInput(task.getBufferAllocator(), { download(target, task) }) {
+            override fun abort() {
+            }
+
+            override fun commit(): TaskReport {
+                return Exec.newTaskReport()
+            }
+        }
+    }
+
+    private fun listTargets(task: PluginTask): List<Target> {
+        val hosts = listHosts(task)
+        val path = getPath(task)
+        val doneTargets = task.getDoneTargets()
+        val ignoreNotFoundHosts = task.getIgnoreNotFoundHosts()
+
+        return hosts.map {
+            val split = it.split(Regex(":"))
+            val host = split[0]
+            val port = if (split.size > 1) split[1].toInt() else task.getDefaultPort()
+            Target(host, port, path)
+        }.filter {
+            !doneTargets.contains(it)
+        }.filter {
+            !ignoreNotFoundHosts || {
+                try {
+                    exists(it, task)
+                } catch (e: IOException) {
+                    log.warn("failed to check the file exists. $it", e)
+                    false
+                }
+            }()
+        }
+    }
+
+    private fun listHosts(task: PluginTask): List<String> {
+        return task.getHostsCommand().transform {
+            execCommand(it).split(task.getHostsSeparator().toRegex())
+        }.or(task.getHosts())
+    }
+
+    private fun getPath(task: PluginTask): String {
+        return task.getPathCommand().transform { execCommand(it) }.or(task.getPath())
+    }
+
+    private fun execCommand(command: String?): String {
+        val pb = ProcessBuilder("sh", "-c", command)    // TODO: windows
+        log.info("Running command $command")
+        val process = pb.start()
+        process.inputStream.use { stream ->
+            BufferedReader(InputStreamReader(stream)).use { brStdout ->
+                val stdout = brStdout.readText()
+
+                val code = process.waitFor()
+                if (code != 0) {
+                    throw IOException("Command finished with non-zero exit code. Exit code is $code")
+                }
+
+                return stdout.trim()
+            }
+        }
+    }
+
+    private fun exists(target: Target, task: PluginTask): Boolean {
+        SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
+            val checkCmd = "ls ${target.path}"    // TODO: windows
+            val timeout = 5/* second */
+            val commandResult = client.execCommand(checkCmd, timeout)
+
+            if (commandResult.status != 0) {
+                log.warn("Remote file not found. $target")
+                return false
+            }
+            return true
+        }
+    }
+
+    private fun download(target: Target, task: PluginTask): InputStream {
+        SSHClient.connect(target.host, target.port, task.getAuthConfig()).use { client ->
+            val stream = ByteArrayOutputStream()
+            client.scpDownload(target.path, stream)
+            return ByteArrayInputStream(stream.toByteArray())
+        }
+    }
+
+    data class Target constructor(
+            @JsonProperty("host") val host: String,
+            @JsonProperty("port") val port: Int,
+            @JsonProperty("path") val path: String) {
+
+        override fun toString(): String {
+            return "$host:$port:$path"
+        }
+    }
+}

data/src/main/kotlin/org/embulk/input/remote/SSHClient.kt

@@ -0,0 +1,74 @@
+package org.embulk.input.remote
+
+import net.schmizz.sshj.DefaultConfig
+import net.schmizz.sshj.transport.verification.PromiscuousVerifier
+import net.schmizz.sshj.xfer.InMemoryDestFile
+import net.schmizz.sshj.xfer.LocalDestFile
+import org.embulk.input.RemoteFileInputPlugin
+import java.io.Closeable
+import java.io.InputStream
+import java.io.OutputStream
+import java.util.concurrent.TimeUnit
+
+class SSHClient private constructor(val client: net.schmizz.sshj.SSHClient) : Closeable {
+    companion object {
+        @JvmStatic
+        fun connect(
+                host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig
+        ): SSHClient {
+            val client = SSHClient(net.schmizz.sshj.SSHClient(DefaultConfig()))
+            client.connectToHost(host, port, authConfig)
+            return client
+        }
+    }
+
+    private fun connectToHost(host: String, port: Int, authConfig: RemoteFileInputPlugin.AuthConfig) {
+        if (authConfig.getSkipHostKeyVerification()) {
+            client.addHostKeyVerifier(PromiscuousVerifier())
+        }
+        client.loadKnownHosts()
+        client.connect(host, port)
+
+        val type = authConfig.getType()
+        val user = authConfig.getUser().or(System.getProperty("user.name"))
+
+        if ("password" == type) {
+            client.authPassword(user, authConfig.getPassword().get())
+        } else if ("public_key" == type) {
+            authConfig.getKeyPath().transform {
+                client.authPublickey(user, it)
+            }.or {
+                client.authPublickey(user)
+            }
+        } else {
+            throw UnsupportedOperationException("Unsupported auth type : " + type)
+        }
+    }
+
+    fun execCommand(command: String, timeoutSecond: Int): CommandResult {
+        client.startSession().use { session ->
+            val cmd = session.exec(command)
+            cmd.join(timeoutSecond.toLong(), TimeUnit.SECONDS)
+            return CommandResult(cmd.exitStatus!!, cmd.inputStream)
+        }
+    }
+
+    fun scpDownload(path: String, stream: OutputStream) {
+        client.useCompression()
+        client.newSCPFileTransfer().download(path, object :InMemoryDestFile() {
+            override fun getOutputStream(): OutputStream {
+                return stream
+            }
+
+            override fun getTargetDirectory(dirname: String): LocalDestFile {
+                return this
+            }
+        })
+    }
+
+    override fun close() {
+        client.close()
+    }
+
+    data class CommandResult(val status: Int, val stdout: InputStream)
+}

data/src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt

@@ -0,0 +1,201 @@
+package org.embulk.input
+
+import ch.qos.logback.classic.Level
+import ch.qos.logback.classic.Logger
+import com.github.dockerjava.core.DockerClientBuilder
+import org.embulk.config.ConfigSource
+import org.embulk.spi.InputPlugin
+import org.embulk.test.EmbulkPluginTest
+import org.embulk.test.ExtendedEmbulkTests
+import org.embulk.test.TestOutputPlugin.assertRecords
+import org.embulk.test.TestingEmbulk
+import org.embulk.test.Utils.record
+import org.hamcrest.CoreMatchers.`is`
+import org.hamcrest.MatcherAssert.assertThat
+import org.junit.Ignore
+import org.junit.Test
+import org.slf4j.LoggerFactory
+import java.util.Arrays
+
+class TestRemoteFileInputPlugin : EmbulkPluginTest() {
+    private val CONTAINER_ID_HOST1 = "embulkinputremote_host1_1"
+    private val CONTAINER_ID_HOST2 = "embulkinputremote_host2_1"
+    private val dockerClient = DockerClientBuilder.getInstance().build()
+
+    override fun setup(builder: TestingEmbulk.Builder) {
+        builder.registerPlugin(InputPlugin::class.java, "remote", RemoteFileInputPlugin::class.java)
+
+        // Setup docker container
+        startContainer(CONTAINER_ID_HOST1)
+        startContainer(CONTAINER_ID_HOST2)
+
+        val logLevel = System.getenv("LOG_LEVEL")
+        if (logLevel != null) {
+            // Set log level
+            val rootLogger = LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME) as Logger
+            rootLogger.level = Level.toLevel(logLevel)
+        }
+    }
+
+    @Test fun loadFromRemote() {
+        runInput(baseConfig())
+        assertRecords(record(1, "user1"))
+    }
+
+    @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
+    @Test fun loadFromRemoteViaPublicKey() {
+        var keyPath: String? = System.getenv("KEY_PATH")
+        if (keyPath == null) {
+            keyPath = "./id_rsa_test"
+        }
+
+        val publicKeyAuth = newConfig().set("auth", newConfig()
+                .set("type", "public_key")
+                .set("key_path", keyPath)
+        )
+        runInput(baseConfig().merge(publicKeyAuth))
+
+        assertRecords(record(1, "user1"))
+    }
+
+    @Test fun testMultiHosts() {
+        val multiHosts = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
+        val config = baseConfig().merge(multiHosts)
+
+        // Run
+        runInput(config)
+        assertRecords(
+                record(1, "user1"),
+                record(2, "user2")
+        )
+    }
+
+    @Test fun loadAllFilesInDirectory() {
+        val directoryPath = newConfig()
+                .set("path", "/mount")
+        val config = baseConfig().merge(directoryPath)
+
+        runInput(config)
+        assertRecords(
+                record(1L, "user1"),
+                record(1L, "command_user1")
+        )
+    }
+
+    @Test fun testDefaultPort() {
+        val defaultPort = newConfig()
+                .set("hosts", listOf("localhost"))
+                .set("default_port", 10022)
+
+        runInput(baseConfig().merge(defaultPort))
+
+        assertRecords(record(1L, "user1"))
+    }
+
+    @Test fun testConfDiff() {
+        val host2Config = newConfig()
+                .set("hosts", listOf("localhost:10023"))
+        var config = baseConfig().merge(host2Config)
+
+        // Run
+        val runResult = runInput(config)
+        assertRecords(record(2, "user2"))
+
+        // Re-run with additional host1
+        val multiHost = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
+        config = baseConfig().merge(multiHost)
+
+        runInput(config, runResult.configDiff)
+
+        assertRecords(record(1, "user1"))
+    }
+
+    @Test fun testResume() {
+        val multiHost = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
+        val config = baseConfig().merge(multiHost)
+
+        // Stop host2 temporarily
+        stopContainer(CONTAINER_ID_HOST2)
+
+        // Run (but will fail)
+        var resumableResult = resume(config)
+
+        assertThat(resumableResult.isSuccessful, `is`(false))
+        assertRecords(record(1, "user1"))
+
+        // Start host2 again
+        startContainer(CONTAINER_ID_HOST2)
+
+        // Resume
+        resumableResult = resume(config, resumableResult.resumeState)
+
+        assertThat(resumableResult.isSuccessful, `is`(true))
+        assertRecords(record(2, "user2"))
+    }
+
+    @Test fun testIgnoreNotFoundHosts() {
+        val ignoreNotFoundHosts = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
+                .set("ignore_not_found_hosts", true)
+        val config = baseConfig().merge(ignoreNotFoundHosts)
+
+        // Stop host2
+        stopContainer(CONTAINER_ID_HOST2)
+
+        // Run (host2 will be ignored)
+        val resumableResult = resume(config)
+
+        assertThat<Boolean>(resumableResult.isSuccessful, `is`(true))
+        assertRecords(record(1, "user1"))
+    }
+
+    @Test fun testCommandOptions() {
+        val ignoreNotFoundHosts = newConfig()
+                .set("hosts_command", "./src/test/resources/script/hosts.sh")
+                .set("hosts_separator", "\n")
+                .set("path_command", "echo '/mount/test_command.csv'")
+        val config = baseConfig().merge(ignoreNotFoundHosts)
+
+        runInput(config)
+
+        assertRecords(
+                record(1, "command_user1"),
+                record(2, "command_user2")
+        )
+    }
+
+    //////////////////////////////
+    // Helpers
+    //////////////////////////////
+
+    private fun baseConfig(): ConfigSource {
+        return ExtendedEmbulkTests.configFromResource("yaml/base.yml")
+    }
+
+    //////////////////////////////
+    // Methods for Docker
+    //////////////////////////////
+
+    private fun stopContainer(containerId: String) {
+        if (isRunning(containerId)) {
+            dockerClient.stopContainerCmd(containerId).exec()
+        }
+    }
+
+    private fun startContainer(containerId: String) {
+        if (!isRunning(containerId)) {
+            dockerClient.startContainerCmd(containerId).exec()
+        }
+    }
+
+    private fun isRunning(containerId: String): Boolean {
+        return dockerClient.listContainersCmd().exec().any { container ->
+            container.names.any { name ->
+                name.contains(containerId)
+            }
+        }
+    }
+}

data/src/test/resources/script/hosts.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+echo -e "localhost:10022\nlocalhost:10023\n\n\n\n"