embulk-input-remote 0.4.1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1705e6d72cf170d8a66a4898d9ab43c5af32d3d
-  data.tar.gz: 83fbc0c400d6dfd49a785105b611fad9116cb37f
+  metadata.gz: 366e8b4b47da07594a9712457893566a0456a200
+  data.tar.gz: 0e7f1777625b862dfc38b1436efe3f83205ef998
 SHA512:
-  metadata.gz: adc9ad79350f5c1f022f922ff6d0f89cc8740e3c2fa5f35ae2def49b1a53372c9d07697703c9ea4f0b16029fa004685ce856f22f45ef4d3418fdddc8ac570f3f
-  data.tar.gz: 0a6bac82d2bd6b81d9206f024345625f47a304172c10af3678f9b80f7e85d7d868457d28cb6d8edf1b5e6f2215f60421fe22966c4b9308893a536523b3b3c70d
+  metadata.gz: 1024e60a974e113ac117fa0e9cf6202db5e2c7362609bf9da10fa576904a34d57178ff852bc8ac40a464a4f395180be7ecf342a49c787588d0022382e73e0bba
+  data.tar.gz: cc19b0b51cd14a12b965f142eec1b530579448e41e1d09aaf611a328d40af28392e6ab93b2beea591e40171581818193003dcc6f6392fb507d7b832fe836088b

data/README.md

@@ -27,6 +27,7 @@ This plugin load data from Remote hosts by SCP
     - **key_path**: Path of your secret key (required when you choose `public_key` type) (string, default: `~/.ssh/id_rsa` or `id_dsa`")
     - **password**: Password of the `user` (required when you choose `password` type) (string)
     - **skip_host_key_verification**: If true, verification of host key will be skipped (boolean, default: false)
+    - **load_known_hosts**: If true, load the known_hosts file (boolean, default: true)
 
 ## Example
 

data/build.gradle

@@ -4,13 +4,13 @@ buildscript {
         maven { url 'http://kamatama41.github.com/maven-repository/repository' }
     }
     dependencies {
-        classpath "com.github.kamatama41:gradle-embulk-plugin:0.1.4"
+        classpath "com.github.kamatama41:gradle-embulk-plugin:0.3.0"
         classpath "net.researchgate:gradle-release:2.5.0"
     }
 }
 
 plugins {
-    id "org.jetbrains.kotlin.jvm" version "1.2.31"
+    id "org.jetbrains.kotlin.jvm" version "1.3.10"
 }
 apply plugin: "com.github.kamatama41.embulk"
 apply plugin: "net.researchgate.release"
@@ -29,14 +29,19 @@ targetCompatibility = 1.8
 
 dependencies {
     compile "org.jetbrains.kotlin:kotlin-stdlib"
-    compile  "com.hierynomus:sshj:0.24.0"
+    compile  "com.hierynomus:sshj:0.26.0"
     compile  "com.jcraft:jzlib:1.1.3"
-    testCompile "com.github.kamatama41:embulk-test-helpers:0.4.0"
+    testCompile "com.github.kamatama41:embulk-test-helpers:0.6.0"
     testCompile "com.github.docker-java:docker-java:3.0.7"
+    testRuntime "org.junit.jupiter:junit-jupiter-engine:5.3.2"
+}
+
+test {
+    useJUnitPlatform()
 }
 
 embulk {
-    version = "0.9.7"
+    version = "0.9.11"
     category = "file-input"
     name = "remote"
     authors = ["Shinichi Ishimura"]

data/gradle.properties

@@ -1 +1 @@
-version=0.4.1
+version=0.4.2

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,5 +1,5 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-5.0-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-4.5.1-all.zip

data/gradlew

@@ -28,7 +28,7 @@ APP_NAME="Gradle"
 APP_BASE_NAME=`basename "$0"`
 
 # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-DEFAULT_JVM_OPTS=""
+DEFAULT_JVM_OPTS='"-Xmx64m"'
 
 # Use the maximum available, or set MAX_FD != -1 to use that value.
 MAX_FD="maximum"

data/gradlew.bat

@@ -1,84 +1,84 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windows variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windows variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/kotlin/org/embulk/input/remote/RemoteFileInputPlugin.kt

@@ -1,7 +1,6 @@
 package org.embulk.input.remote
 
 import com.fasterxml.jackson.annotation.JsonProperty
-import com.google.common.base.Optional
 import org.embulk.config.Config
 import org.embulk.config.ConfigDefault
 import org.embulk.config.ConfigDiff
@@ -21,6 +20,7 @@ import java.io.ByteArrayOutputStream
 import java.io.IOException
 import java.io.InputStream
 import java.io.InputStreamReader
+import java.util.Optional
 
 class RemoteFileInputPlugin : FileInputPlugin {
     interface PluginTask : Task {
@@ -85,6 +85,10 @@ class RemoteFileInputPlugin : FileInputPlugin {
         @get:Config("skip_host_key_verification")
         @get:ConfigDefault("false")
         val skipHostKeyVerification: Boolean
+
+        @get:Config("load_known_hosts")
+        @get:ConfigDefault("true")
+        val loadKnownHosts: Boolean
     }
 
     private val log = getLogger()
@@ -149,13 +153,13 @@ class RemoteFileInputPlugin : FileInputPlugin {
     }
 
     private fun listHosts(task: PluginTask): List<String> {
-        return task.hostsCommand.transform {
+        return task.hostsCommand.map {
             execCommand(it).split(task.hostsSeparator.toRegex())
-        }.or(task.hosts)
+        }.orElse(task.hosts)
     }
 
     private fun getPath(task: PluginTask): String {
-        return task.pathCommand.transform { execCommand(it) }.or(task.path)
+        return task.pathCommand.map { execCommand(it) }.orElse(task.path)
     }
 
     private fun execCommand(command: String?): String {

data/src/main/kotlin/org/embulk/input/remote/SSHClient.kt

@@ -23,20 +23,22 @@ class SSHClient private constructor(val client: SSHJ) : Closeable {
         if (authConfig.skipHostKeyVerification) {
             client.addHostKeyVerifier(PromiscuousVerifier())
         }
-        client.loadKnownHosts()
+        if (authConfig.loadKnownHosts) {
+            client.loadKnownHosts()
+        }
         client.connect(host, port)
 
         val type = authConfig.type
-        val user = authConfig.user.or(System.getProperty("user.name"))
+        val user = authConfig.user.orElse(System.getProperty("user.name"))
 
         when (type) {
             "password" -> {
                 client.authPassword(user, authConfig.password.get())
             }
             "public_key" -> {
-                authConfig.keyPath.transform {
+                authConfig.keyPath.map {
                     client.authPublickey(user, it)
-                }.or {
+                }.orElseGet {
                     client.authPublickey(user)
                 }
             }

data/src/test/kotlin/org/embulk/input/remote/TestRemoteFileInputPlugin.kt

@@ -5,22 +5,20 @@ import ch.qos.logback.classic.Logger
 import com.github.dockerjava.core.DockerClientBuilder
 import org.embulk.config.ConfigSource
 import org.embulk.test.EmbulkPluginTest
+import org.embulk.test.EmbulkTest
 import org.embulk.test.TestOutputPlugin.Matcher.assertRecords
 import org.embulk.test.configFromResource
 import org.embulk.test.record
-import org.embulk.test.registerPlugin
 import org.embulk.test.set
 import org.hamcrest.CoreMatchers.`is`
 import org.hamcrest.MatcherAssert.assertThat
-import org.junit.Before
-import org.junit.Ignore
-import org.junit.Test
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
 import org.slf4j.LoggerFactory
 
+@EmbulkTest(value = [RemoteFileInputPlugin::class])
 class TestRemoteFileInputPlugin : EmbulkPluginTest() {
-    @Before fun setup() {
-        builder.registerPlugin(RemoteFileInputPlugin::class)
-
+    @BeforeEach fun setup() {
         // Setup docker container
         startContainer(CONTAINER_ID_HOST1)
         startContainer(CONTAINER_ID_HOST2)

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.4.2
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-04-26 00:00:00.000000000 Z
+date: 2019-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   name: bundler
@@ -21,13 +21,13 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
   name: rake
@@ -35,7 +35,7 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.0'
 description: Reads files stored on Remote hosts by SCP.
@@ -45,11 +45,20 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .circleci/config.yml
-- .gitignore
+- ".circleci/config.yml"
+- ".gitignore"
 - LICENSE.txt
 - README.md
 - build.gradle
+- classpath/annotations-13.0.jar
+- classpath/bcpkix-jdk15on-1.60.jar
+- classpath/bcprov-jdk15on-1.60.jar
+- classpath/eddsa-0.2.0.jar
+- classpath/embulk-input-remote-0.4.2.jar
+- classpath/jzlib-1.1.3.jar
+- classpath/kotlin-stdlib-1.3.10.jar
+- classpath/kotlin-stdlib-common-1.3.10.jar
+- classpath/sshj-0.26.0.jar
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - docker-compose.yml
@@ -69,14 +78,6 @@ files:
 - src/test/resources/input/host2/test_command.csv
 - src/test/resources/script/hosts.sh
 - src/test/resources/yaml/base.yml
-- classpath/embulk-input-remote-0.4.1.jar
-- classpath/bcpkix-jdk15on-1.57.jar
-- classpath/kotlin-stdlib-1.2.31.jar
-- classpath/sshj-0.24.0.jar
-- classpath/jzlib-1.1.3.jar
-- classpath/eddsa-0.2.0.jar
-- classpath/annotations-13.0.jar
-- classpath/bcprov-jdk15on-1.57.jar
 homepage: https://github.com/kamatama41/embulk-input-remote
 licenses:
 - MIT
@@ -87,17 +88,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.1.9
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: Remote file input plugin for Embulk