embulk-input-remote 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +20 -0
- data/Dockerfile +16 -0
- data/README.md +3 -0
- data/build.gradle +14 -4
- data/classpath/embulk-input-remote-0.1.10.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +26 -26
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java +18 -14
- data/src/main/java/org/embulk/input/remote/SSHClient.java +6 -6
- data/src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java +32 -28
- data/src/test/resources/expect/test01.csv +2 -0
- data/src/test/resources/input/test01.csv +2 -0
- data/src/test/resources/yaml/test01.yml +15 -0
- metadata +8 -3
- data/classpath/embulk-input-remote-0.1.9.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05d25bac2bc3ef17ff27004d9fce3e6317b8be80
|
4
|
+
data.tar.gz: 502556efd3275e388fc7ebb072dacc9bfa524231
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8a99a040b1900a2458787b658699ea6b449b5c2d95de42b23554b44a152a88f2ef73efa2a3d53299e2a31e51d3147b187166905901c75e78c2787dfc767c920d
|
7
|
+
data.tar.gz: f7203448e89983ca048cd4c85dbc8a503dbbda92bf49f793f5f46ce81c1d4291a3d2e86f0a693e84f515d7731a51a64fe94b57ff71c978c6bc271736251f8733
|
data/.travis.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
sudo: required
|
2
|
+
|
3
|
+
language: java
|
4
|
+
|
5
|
+
jdk:
|
6
|
+
- oraclejdk8
|
7
|
+
|
8
|
+
env:
|
9
|
+
- YAML_TEST01=$PWD/src/test/resources/yaml/test01.yml
|
10
|
+
|
11
|
+
services:
|
12
|
+
- docker
|
13
|
+
|
14
|
+
before_install:
|
15
|
+
- docker build -t eg_sshd .
|
16
|
+
- docker run -d -p 10022:22 -v $PWD/src/test/resources/input:/mount --name test_sshd eg_sshd
|
17
|
+
- docker ps -a
|
18
|
+
|
19
|
+
script:
|
20
|
+
- ./gradlew --info check
|
data/Dockerfile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# ref: https://docs.docker.com/engine/examples/running_ssh_service/
|
2
|
+
FROM ubuntu:16.04
|
3
|
+
|
4
|
+
RUN apt-get update && apt-get install -y openssh-server
|
5
|
+
RUN mkdir /var/run/sshd
|
6
|
+
RUN echo 'root:screencast' | chpasswd
|
7
|
+
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
|
8
|
+
|
9
|
+
# SSH login fix. Otherwise user is kicked off after login
|
10
|
+
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
|
11
|
+
|
12
|
+
ENV NOTVISIBLE "in users profile"
|
13
|
+
RUN echo "export VISIBLE=now" >> /etc/profile
|
14
|
+
|
15
|
+
EXPOSE 22
|
16
|
+
CMD ["/usr/sbin/sshd", "-D"]
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[](https://travis-ci.org/kamatama41/tfenv)
|
2
|
+
|
1
3
|
# Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
|
2
4
|
|
3
5
|
This plugin load data from Remote hosts by SCP
|
@@ -13,6 +15,7 @@ This plugin load data from Remote hosts by SCP
|
|
13
15
|
- **hosts**: Target hosts (list, default: [])
|
14
16
|
- **hosts_command**: Command for getting hosts(Windows not supported). If given the option, "hosts" is overwritten. (string, default: null)
|
15
17
|
- **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
|
18
|
+
- **port**: Port number for SSH (integer, default: 22)
|
16
19
|
- **path**: Path of remote host (File or Directory) (string, default: "")
|
17
20
|
- **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
|
18
21
|
- **ignore_not_found_hosts**: If the option is true, Hosts which meet the following conditions are skipped. (Means they are not included into resume target.) (boolean, default: false)
|
data/build.gradle
CHANGED
@@ -4,6 +4,11 @@ plugins {
|
|
4
4
|
id "java"
|
5
5
|
id "idea"
|
6
6
|
}
|
7
|
+
|
8
|
+
compileJava {
|
9
|
+
options.compilerArgs = ['-Xlint:all']
|
10
|
+
}
|
11
|
+
|
7
12
|
import com.github.jrubygradle.JRubyExec
|
8
13
|
repositories {
|
9
14
|
mavenCentral()
|
@@ -13,14 +18,19 @@ configurations {
|
|
13
18
|
provided
|
14
19
|
}
|
15
20
|
|
16
|
-
version = "0.1.
|
21
|
+
version = "0.1.10"
|
17
22
|
|
18
23
|
dependencies {
|
19
|
-
compile "org.embulk:embulk-core:0.
|
24
|
+
compile "org.embulk:embulk-core:0.8.15"
|
20
25
|
compile "com.hierynomus:sshj:0.15.0"
|
21
26
|
compile "com.jcraft:jzlib:1.1.3"
|
22
|
-
provided "org.embulk:embulk-core:0.
|
23
|
-
testCompile "
|
27
|
+
provided "org.embulk:embulk-core:0.8.15"
|
28
|
+
testCompile "org.embulk:embulk-standards:0.8.15"
|
29
|
+
testCompile "org.embulk:embulk-test:0.8.15"
|
30
|
+
// for local testing
|
31
|
+
// testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
|
32
|
+
// testCompile 'junit:junit:4.12'
|
33
|
+
// testCompile 'org.hamcrest:hamcrest-library:1.3'
|
24
34
|
}
|
25
35
|
|
26
36
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Sat Dec 17 23:39:37 JST 2016
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip
|
data/gradlew
CHANGED
@@ -6,12 +6,30 @@
|
|
6
6
|
##
|
7
7
|
##############################################################################
|
8
8
|
|
9
|
-
#
|
10
|
-
|
9
|
+
# Attempt to set APP_HOME
|
10
|
+
# Resolve links: $0 may be a link
|
11
|
+
PRG="$0"
|
12
|
+
# Need this for relative symlinks.
|
13
|
+
while [ -h "$PRG" ] ; do
|
14
|
+
ls=`ls -ld "$PRG"`
|
15
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
16
|
+
if expr "$link" : '/.*' > /dev/null; then
|
17
|
+
PRG="$link"
|
18
|
+
else
|
19
|
+
PRG=`dirname "$PRG"`"/$link"
|
20
|
+
fi
|
21
|
+
done
|
22
|
+
SAVED="`pwd`"
|
23
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
24
|
+
APP_HOME="`pwd -P`"
|
25
|
+
cd "$SAVED" >/dev/null
|
11
26
|
|
12
27
|
APP_NAME="Gradle"
|
13
28
|
APP_BASE_NAME=`basename "$0"`
|
14
29
|
|
30
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
31
|
+
DEFAULT_JVM_OPTS=""
|
32
|
+
|
15
33
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
34
|
MAX_FD="maximum"
|
17
35
|
|
@@ -30,6 +48,7 @@ die ( ) {
|
|
30
48
|
cygwin=false
|
31
49
|
msys=false
|
32
50
|
darwin=false
|
51
|
+
nonstop=false
|
33
52
|
case "`uname`" in
|
34
53
|
CYGWIN* )
|
35
54
|
cygwin=true
|
@@ -40,31 +59,11 @@ case "`uname`" in
|
|
40
59
|
MINGW* )
|
41
60
|
msys=true
|
42
61
|
;;
|
62
|
+
NONSTOP* )
|
63
|
+
nonstop=true
|
64
|
+
;;
|
43
65
|
esac
|
44
66
|
|
45
|
-
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
46
|
-
if $cygwin ; then
|
47
|
-
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
48
|
-
fi
|
49
|
-
|
50
|
-
# Attempt to set APP_HOME
|
51
|
-
# Resolve links: $0 may be a link
|
52
|
-
PRG="$0"
|
53
|
-
# Need this for relative symlinks.
|
54
|
-
while [ -h "$PRG" ] ; do
|
55
|
-
ls=`ls -ld "$PRG"`
|
56
|
-
link=`expr "$ls" : '.*-> \(.*\)$'`
|
57
|
-
if expr "$link" : '/.*' > /dev/null; then
|
58
|
-
PRG="$link"
|
59
|
-
else
|
60
|
-
PRG=`dirname "$PRG"`"/$link"
|
61
|
-
fi
|
62
|
-
done
|
63
|
-
SAVED="`pwd`"
|
64
|
-
cd "`dirname \"$PRG\"`/" >&-
|
65
|
-
APP_HOME="`pwd -P`"
|
66
|
-
cd "$SAVED" >&-
|
67
|
-
|
68
67
|
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
69
68
|
|
70
69
|
# Determine the Java command to use to start the JVM.
|
@@ -90,7 +89,7 @@ location of your Java installation."
|
|
90
89
|
fi
|
91
90
|
|
92
91
|
# Increase the maximum file descriptors if we can.
|
93
|
-
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
92
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
94
93
|
MAX_FD_LIMIT=`ulimit -H -n`
|
95
94
|
if [ $? -eq 0 ] ; then
|
96
95
|
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
@@ -114,6 +113,7 @@ fi
|
|
114
113
|
if $cygwin ; then
|
115
114
|
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
116
115
|
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
116
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
117
117
|
|
118
118
|
# We build the pattern for arguments to be converted via cygpath
|
119
119
|
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
data/gradlew.bat
CHANGED
@@ -1,90 +1,90 @@
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
2
|
-
@rem ##########################################################################
|
3
|
-
@rem
|
4
|
-
@rem Gradle startup script for Windows
|
5
|
-
@rem
|
6
|
-
@rem ##########################################################################
|
7
|
-
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
10
|
-
|
11
|
-
|
12
|
-
set
|
13
|
-
|
14
|
-
set DIRNAME
|
15
|
-
|
16
|
-
|
17
|
-
set
|
18
|
-
|
19
|
-
@rem Find java.exe
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
-
|
22
|
-
set JAVA_EXE=java.exe
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
25
|
-
|
26
|
-
echo.
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
-
echo.
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
-
echo location of your Java installation.
|
31
|
-
|
32
|
-
goto fail
|
33
|
-
|
34
|
-
:findJavaFromJavaHome
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
-
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
39
|
-
|
40
|
-
echo.
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
-
echo.
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
-
echo location of your Java installation.
|
45
|
-
|
46
|
-
goto fail
|
47
|
-
|
48
|
-
:init
|
49
|
-
@rem Get command-line arguments, handling
|
50
|
-
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
-
|
54
|
-
:win9xME_args
|
55
|
-
@rem Slurp the command line arguments.
|
56
|
-
set CMD_LINE_ARGS=
|
57
|
-
set _SKIP=2
|
58
|
-
|
59
|
-
:win9xME_args_slurp
|
60
|
-
if "x%~1" == "x" goto execute
|
61
|
-
|
62
|
-
set CMD_LINE_ARGS=%*
|
63
|
-
goto execute
|
64
|
-
|
65
|
-
:4NT_args
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
-
set CMD_LINE_ARGS=%$
|
68
|
-
|
69
|
-
:execute
|
70
|
-
@rem Setup the command line
|
71
|
-
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
-
|
74
|
-
@rem Execute Gradle
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
-
|
77
|
-
:end
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
-
|
81
|
-
:fail
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
-
rem the _cmd.exe /c_ return code!
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
-
exit /b 1
|
86
|
-
|
87
|
-
:mainEnd
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
89
|
-
|
90
|
-
:omega
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS=
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
+
|
54
|
+
:win9xME_args
|
55
|
+
@rem Slurp the command line arguments.
|
56
|
+
set CMD_LINE_ARGS=
|
57
|
+
set _SKIP=2
|
58
|
+
|
59
|
+
:win9xME_args_slurp
|
60
|
+
if "x%~1" == "x" goto execute
|
61
|
+
|
62
|
+
set CMD_LINE_ARGS=%*
|
63
|
+
goto execute
|
64
|
+
|
65
|
+
:4NT_args
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
+
set CMD_LINE_ARGS=%$
|
68
|
+
|
69
|
+
:execute
|
70
|
+
@rem Setup the command line
|
71
|
+
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
+
|
74
|
+
@rem Execute Gradle
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
+
|
77
|
+
:end
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
+
|
81
|
+
:fail
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
+
rem the _cmd.exe /c_ return code!
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
+
exit /b 1
|
86
|
+
|
87
|
+
:mainEnd
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
89
|
+
|
90
|
+
:omega
|
@@ -40,44 +40,48 @@ public class RemoteFileInputPlugin
|
|
40
40
|
extends Task {
|
41
41
|
@Config("hosts")
|
42
42
|
@ConfigDefault("[]")
|
43
|
-
|
43
|
+
List<String> getHosts();
|
44
44
|
|
45
45
|
@Config("hosts_command")
|
46
46
|
@ConfigDefault("null")
|
47
|
-
|
47
|
+
Optional<String> getHostsCommand();
|
48
48
|
|
49
49
|
@Config("hosts_separator")
|
50
50
|
@ConfigDefault("\" \"")
|
51
|
-
|
51
|
+
String getHostsSeparator();
|
52
|
+
|
53
|
+
@Config("port")
|
54
|
+
@ConfigDefault("22")
|
55
|
+
int getPort();
|
52
56
|
|
53
57
|
@Config("path")
|
54
58
|
@ConfigDefault("\"\"")
|
55
|
-
|
59
|
+
String getPath();
|
56
60
|
|
57
61
|
@Config("path_command")
|
58
62
|
@ConfigDefault("null")
|
59
|
-
|
63
|
+
Optional<String> getPathCommand();
|
60
64
|
|
61
65
|
@Config("auth")
|
62
66
|
@ConfigDefault("{}")
|
63
|
-
|
67
|
+
Map<String, String> getAuth();
|
64
68
|
|
65
69
|
@Config("ignore_not_found_hosts")
|
66
70
|
@ConfigDefault("false")
|
67
|
-
|
71
|
+
boolean getIgnoreNotFoundHosts();
|
68
72
|
|
69
73
|
@Config("last_target")
|
70
74
|
@ConfigDefault("null")
|
71
|
-
|
75
|
+
Optional<Target> getLastTarget();
|
72
76
|
|
73
|
-
|
77
|
+
void setLastTarget(Optional<Target> lastTarget);
|
74
78
|
|
75
|
-
|
79
|
+
List<Target> getTargets();
|
76
80
|
|
77
|
-
|
81
|
+
void setTargets(List<Target> targets);
|
78
82
|
|
79
83
|
@ConfigInject
|
80
|
-
|
84
|
+
BufferAllocator getBufferAllocator();
|
81
85
|
}
|
82
86
|
|
83
87
|
private final Logger log = Exec.getLogger(getClass());
|
@@ -219,7 +223,7 @@ public class RemoteFileInputPlugin
|
|
219
223
|
|
220
224
|
private boolean exists(Target target, PluginTask task) throws IOException {
|
221
225
|
try (SSHClient client = SSHClient.getInstance()) {
|
222
|
-
client.connect(target.getHost(), task.getAuth());
|
226
|
+
client.connect(target.getHost(), task.getPort(), task.getAuth());
|
223
227
|
|
224
228
|
final String checkCmd = "ls " + target.getPath(); // TODO: windows
|
225
229
|
final int timeout = 5/* second */;
|
@@ -236,7 +240,7 @@ public class RemoteFileInputPlugin
|
|
236
240
|
|
237
241
|
private InputStream download(Target target, PluginTask task) throws IOException {
|
238
242
|
try (SSHClient client = SSHClient.getInstance()) {
|
239
|
-
client.connect(target.getHost(), task.getAuth());
|
243
|
+
client.connect(target.getHost(), task.getPort(), task.getAuth());
|
240
244
|
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
|
241
245
|
client.scpDownload(target.getPath(), stream);
|
242
246
|
return new ByteArrayInputStream(stream.toByteArray());
|
@@ -14,6 +14,7 @@ import java.io.Closeable;
|
|
14
14
|
import java.io.IOException;
|
15
15
|
import java.io.InputStream;
|
16
16
|
import java.io.OutputStream;
|
17
|
+
import java.util.Arrays;
|
17
18
|
import java.util.Map;
|
18
19
|
import java.util.concurrent.TimeUnit;
|
19
20
|
|
@@ -25,27 +26,26 @@ public class SSHClient implements Closeable {
|
|
25
26
|
return new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig(){
|
26
27
|
@Override
|
27
28
|
protected void initSignatureFactories() {
|
28
|
-
setSignatureFactories(
|
29
|
+
setSignatureFactories(Arrays.asList(
|
29
30
|
new SignatureRSA.Factory(),
|
30
31
|
new SignatureECDSA.Factory(),
|
31
32
|
new SignatureDSA.Factory(),
|
32
33
|
new SignatureEdDSA.Factory()
|
33
|
-
);
|
34
|
+
));
|
34
35
|
}
|
35
36
|
}));
|
36
37
|
}
|
37
38
|
|
38
|
-
|
39
|
-
SSHClient(net.schmizz.sshj.SSHClient client) {
|
39
|
+
private SSHClient(net.schmizz.sshj.SSHClient client) {
|
40
40
|
this.client = client;
|
41
41
|
}
|
42
42
|
|
43
|
-
public void connect(String host, Map<String, String> authConfig) throws IOException {
|
43
|
+
public void connect(String host, int port, Map<String, String> authConfig) throws IOException {
|
44
44
|
if (Boolean.valueOf(authConfig.get("skip_host_key_verification"))) {
|
45
45
|
client.addHostKeyVerifier(new PromiscuousVerifier());
|
46
46
|
}
|
47
47
|
client.loadKnownHosts();
|
48
|
-
client.connect(host);
|
48
|
+
client.connect(host, port);
|
49
49
|
|
50
50
|
final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";
|
51
51
|
final String user = authConfig.get("user") != null ? authConfig.get("user") : System.getProperty("user.name");
|
@@ -1,34 +1,38 @@
|
|
1
1
|
package org.embulk.input;
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
import org.embulk.config.ConfigSource;
|
4
|
+
import org.embulk.spi.InputPlugin;
|
5
|
+
import org.embulk.test.EmbulkTests;
|
6
|
+
import org.embulk.test.TestingEmbulk;
|
7
|
+
import org.junit.Rule;
|
8
|
+
import org.junit.Test;
|
9
|
+
|
10
|
+
import java.nio.file.Path;
|
11
|
+
|
12
|
+
import static org.embulk.test.EmbulkTests.readResource;
|
13
|
+
import static org.embulk.test.EmbulkTests.readSortedFile;
|
14
|
+
import static org.hamcrest.Matchers.is;
|
15
|
+
import static org.junit.Assert.assertThat;
|
14
16
|
|
15
17
|
public class TestRemoteFileInputPlugin
|
16
18
|
{
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
19
|
+
@Rule
|
20
|
+
public TestingEmbulk embulk = TestingEmbulk
|
21
|
+
.builder()
|
22
|
+
.registerPlugin(InputPlugin.class, "remote", RemoteFileInputPlugin.class)
|
23
|
+
.build();
|
24
|
+
|
25
|
+
@Test
|
26
|
+
public void loadFromRemote() throws Exception
|
27
|
+
{
|
28
|
+
ConfigSource config = EmbulkTests.config("YAML_TEST01");
|
29
|
+
Path out = embulk.createTempFile("csv");
|
30
|
+
|
31
|
+
embulk.runInput(config, out);
|
32
|
+
|
33
|
+
assertThat(
|
34
|
+
readSortedFile(out),
|
35
|
+
is(readResource("expect/test01.csv")));
|
36
|
+
}
|
37
|
+
|
34
38
|
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
type: remote
|
2
|
+
hosts:
|
3
|
+
- localhost
|
4
|
+
port: 10022
|
5
|
+
path: /mount/test01.csv
|
6
|
+
auth:
|
7
|
+
user: root
|
8
|
+
type: password
|
9
|
+
password: screencast
|
10
|
+
skip_host_key_verification: true
|
11
|
+
parser:
|
12
|
+
type: csv
|
13
|
+
columns:
|
14
|
+
- {name: id, type: long}
|
15
|
+
- {name: username, type: string}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-remote
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi Ishimura
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,8 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
50
|
+
- Dockerfile
|
49
51
|
- LICENSE.txt
|
50
52
|
- README.md
|
51
53
|
- build.gradle
|
@@ -59,10 +61,13 @@ files:
|
|
59
61
|
- src/main/java/org/embulk/input/RemoteFileInputPlugin.java
|
60
62
|
- src/main/java/org/embulk/input/remote/SSHClient.java
|
61
63
|
- src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
|
64
|
+
- src/test/resources/expect/test01.csv
|
65
|
+
- src/test/resources/input/test01.csv
|
66
|
+
- src/test/resources/yaml/test01.yml
|
62
67
|
- classpath/bcpkix-jdk15on-1.51.jar
|
63
68
|
- classpath/bcprov-jdk15on-1.51.jar
|
64
69
|
- classpath/ecc-25519-java-1.0.1.jar
|
65
|
-
- classpath/embulk-input-remote-0.1.
|
70
|
+
- classpath/embulk-input-remote-0.1.10.jar
|
66
71
|
- classpath/jzlib-1.1.3.jar
|
67
72
|
- classpath/sshj-0.15.0.jar
|
68
73
|
homepage: https://github.com/kamatama41/embulk-input-remote
|
Binary file
|