embulk-input-remote 0.1.9 → 0.1.10
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +20 -0
- data/Dockerfile +16 -0
- data/README.md +3 -0
- data/build.gradle +14 -4
- data/classpath/embulk-input-remote-0.1.10.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/gradlew +26 -26
- data/gradlew.bat +90 -90
- data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java +18 -14
- data/src/main/java/org/embulk/input/remote/SSHClient.java +6 -6
- data/src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java +32 -28
- data/src/test/resources/expect/test01.csv +2 -0
- data/src/test/resources/input/test01.csv +2 -0
- data/src/test/resources/yaml/test01.yml +15 -0
- metadata +8 -3
- data/classpath/embulk-input-remote-0.1.9.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 05d25bac2bc3ef17ff27004d9fce3e6317b8be80
|
4
|
+
data.tar.gz: 502556efd3275e388fc7ebb072dacc9bfa524231
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8a99a040b1900a2458787b658699ea6b449b5c2d95de42b23554b44a152a88f2ef73efa2a3d53299e2a31e51d3147b187166905901c75e78c2787dfc767c920d
|
7
|
+
data.tar.gz: f7203448e89983ca048cd4c85dbc8a503dbbda92bf49f793f5f46ce81c1d4291a3d2e86f0a693e84f515d7731a51a64fe94b57ff71c978c6bc271736251f8733
|
data/.travis.yml
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
sudo: required
|
2
|
+
|
3
|
+
language: java
|
4
|
+
|
5
|
+
jdk:
|
6
|
+
- oraclejdk8
|
7
|
+
|
8
|
+
env:
|
9
|
+
- YAML_TEST01=$PWD/src/test/resources/yaml/test01.yml
|
10
|
+
|
11
|
+
services:
|
12
|
+
- docker
|
13
|
+
|
14
|
+
before_install:
|
15
|
+
- docker build -t eg_sshd .
|
16
|
+
- docker run -d -p 10022:22 -v $PWD/src/test/resources/input:/mount --name test_sshd eg_sshd
|
17
|
+
- docker ps -a
|
18
|
+
|
19
|
+
script:
|
20
|
+
- ./gradlew --info check
|
data/Dockerfile
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
# ref: https://docs.docker.com/engine/examples/running_ssh_service/
|
2
|
+
FROM ubuntu:16.04
|
3
|
+
|
4
|
+
RUN apt-get update && apt-get install -y openssh-server
|
5
|
+
RUN mkdir /var/run/sshd
|
6
|
+
RUN echo 'root:screencast' | chpasswd
|
7
|
+
RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
|
8
|
+
|
9
|
+
# SSH login fix. Otherwise user is kicked off after login
|
10
|
+
RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
|
11
|
+
|
12
|
+
ENV NOTVISIBLE "in users profile"
|
13
|
+
RUN echo "export VISIBLE=now" >> /etc/profile
|
14
|
+
|
15
|
+
EXPOSE 22
|
16
|
+
CMD ["/usr/sbin/sshd", "-D"]
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/kamatama41/tfenv.svg?branch=master)](https://travis-ci.org/kamatama41/tfenv)
|
2
|
+
|
1
3
|
# Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
|
2
4
|
|
3
5
|
This plugin load data from Remote hosts by SCP
|
@@ -13,6 +15,7 @@ This plugin load data from Remote hosts by SCP
|
|
13
15
|
- **hosts**: Target hosts (list, default: [])
|
14
16
|
- **hosts_command**: Command for getting hosts(Windows not supported). If given the option, "hosts" is overwritten. (string, default: null)
|
15
17
|
- **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
|
18
|
+
- **port**: Port number for SSH (integer, default: 22)
|
16
19
|
- **path**: Path of remote host (File or Directory) (string, default: "")
|
17
20
|
- **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
|
18
21
|
- **ignore_not_found_hosts**: If the option is true, Hosts which meet the following conditions are skipped. (Means they are not included into resume target.) (boolean, default: false)
|
data/build.gradle
CHANGED
@@ -4,6 +4,11 @@ plugins {
|
|
4
4
|
id "java"
|
5
5
|
id "idea"
|
6
6
|
}
|
7
|
+
|
8
|
+
compileJava {
|
9
|
+
options.compilerArgs = ['-Xlint:all']
|
10
|
+
}
|
11
|
+
|
7
12
|
import com.github.jrubygradle.JRubyExec
|
8
13
|
repositories {
|
9
14
|
mavenCentral()
|
@@ -13,14 +18,19 @@ configurations {
|
|
13
18
|
provided
|
14
19
|
}
|
15
20
|
|
16
|
-
version = "0.1.
|
21
|
+
version = "0.1.10"
|
17
22
|
|
18
23
|
dependencies {
|
19
|
-
compile "org.embulk:embulk-core:0.
|
24
|
+
compile "org.embulk:embulk-core:0.8.15"
|
20
25
|
compile "com.hierynomus:sshj:0.15.0"
|
21
26
|
compile "com.jcraft:jzlib:1.1.3"
|
22
|
-
provided "org.embulk:embulk-core:0.
|
23
|
-
testCompile "
|
27
|
+
provided "org.embulk:embulk-core:0.8.15"
|
28
|
+
testCompile "org.embulk:embulk-standards:0.8.15"
|
29
|
+
testCompile "org.embulk:embulk-test:0.8.15"
|
30
|
+
// for local testing
|
31
|
+
// testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
|
32
|
+
// testCompile 'junit:junit:4.12'
|
33
|
+
// testCompile 'org.hamcrest:hamcrest-library:1.3'
|
24
34
|
}
|
25
35
|
|
26
36
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Sat Dec 17 23:39:37 JST 2016
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-2.14.1-bin.zip
|
data/gradlew
CHANGED
@@ -6,12 +6,30 @@
|
|
6
6
|
##
|
7
7
|
##############################################################################
|
8
8
|
|
9
|
-
#
|
10
|
-
|
9
|
+
# Attempt to set APP_HOME
|
10
|
+
# Resolve links: $0 may be a link
|
11
|
+
PRG="$0"
|
12
|
+
# Need this for relative symlinks.
|
13
|
+
while [ -h "$PRG" ] ; do
|
14
|
+
ls=`ls -ld "$PRG"`
|
15
|
+
link=`expr "$ls" : '.*-> \(.*\)$'`
|
16
|
+
if expr "$link" : '/.*' > /dev/null; then
|
17
|
+
PRG="$link"
|
18
|
+
else
|
19
|
+
PRG=`dirname "$PRG"`"/$link"
|
20
|
+
fi
|
21
|
+
done
|
22
|
+
SAVED="`pwd`"
|
23
|
+
cd "`dirname \"$PRG\"`/" >/dev/null
|
24
|
+
APP_HOME="`pwd -P`"
|
25
|
+
cd "$SAVED" >/dev/null
|
11
26
|
|
12
27
|
APP_NAME="Gradle"
|
13
28
|
APP_BASE_NAME=`basename "$0"`
|
14
29
|
|
30
|
+
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
31
|
+
DEFAULT_JVM_OPTS=""
|
32
|
+
|
15
33
|
# Use the maximum available, or set MAX_FD != -1 to use that value.
|
16
34
|
MAX_FD="maximum"
|
17
35
|
|
@@ -30,6 +48,7 @@ die ( ) {
|
|
30
48
|
cygwin=false
|
31
49
|
msys=false
|
32
50
|
darwin=false
|
51
|
+
nonstop=false
|
33
52
|
case "`uname`" in
|
34
53
|
CYGWIN* )
|
35
54
|
cygwin=true
|
@@ -40,31 +59,11 @@ case "`uname`" in
|
|
40
59
|
MINGW* )
|
41
60
|
msys=true
|
42
61
|
;;
|
62
|
+
NONSTOP* )
|
63
|
+
nonstop=true
|
64
|
+
;;
|
43
65
|
esac
|
44
66
|
|
45
|
-
# For Cygwin, ensure paths are in UNIX format before anything is touched.
|
46
|
-
if $cygwin ; then
|
47
|
-
[ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"`
|
48
|
-
fi
|
49
|
-
|
50
|
-
# Attempt to set APP_HOME
|
51
|
-
# Resolve links: $0 may be a link
|
52
|
-
PRG="$0"
|
53
|
-
# Need this for relative symlinks.
|
54
|
-
while [ -h "$PRG" ] ; do
|
55
|
-
ls=`ls -ld "$PRG"`
|
56
|
-
link=`expr "$ls" : '.*-> \(.*\)$'`
|
57
|
-
if expr "$link" : '/.*' > /dev/null; then
|
58
|
-
PRG="$link"
|
59
|
-
else
|
60
|
-
PRG=`dirname "$PRG"`"/$link"
|
61
|
-
fi
|
62
|
-
done
|
63
|
-
SAVED="`pwd`"
|
64
|
-
cd "`dirname \"$PRG\"`/" >&-
|
65
|
-
APP_HOME="`pwd -P`"
|
66
|
-
cd "$SAVED" >&-
|
67
|
-
|
68
67
|
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
|
69
68
|
|
70
69
|
# Determine the Java command to use to start the JVM.
|
@@ -90,7 +89,7 @@ location of your Java installation."
|
|
90
89
|
fi
|
91
90
|
|
92
91
|
# Increase the maximum file descriptors if we can.
|
93
|
-
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
|
92
|
+
if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then
|
94
93
|
MAX_FD_LIMIT=`ulimit -H -n`
|
95
94
|
if [ $? -eq 0 ] ; then
|
96
95
|
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
|
@@ -114,6 +113,7 @@ fi
|
|
114
113
|
if $cygwin ; then
|
115
114
|
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
|
116
115
|
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
|
116
|
+
JAVACMD=`cygpath --unix "$JAVACMD"`
|
117
117
|
|
118
118
|
# We build the pattern for arguments to be converted via cygpath
|
119
119
|
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
|
data/gradlew.bat
CHANGED
@@ -1,90 +1,90 @@
|
|
1
|
-
@if "%DEBUG%" == "" @echo off
|
2
|
-
@rem ##########################################################################
|
3
|
-
@rem
|
4
|
-
@rem Gradle startup script for Windows
|
5
|
-
@rem
|
6
|
-
@rem ##########################################################################
|
7
|
-
|
8
|
-
@rem Set local scope for the variables with windows NT shell
|
9
|
-
if "%OS%"=="Windows_NT" setlocal
|
10
|
-
|
11
|
-
|
12
|
-
set
|
13
|
-
|
14
|
-
set DIRNAME
|
15
|
-
|
16
|
-
|
17
|
-
set
|
18
|
-
|
19
|
-
@rem Find java.exe
|
20
|
-
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
-
|
22
|
-
set JAVA_EXE=java.exe
|
23
|
-
%JAVA_EXE% -version >NUL 2>&1
|
24
|
-
if "%ERRORLEVEL%" == "0" goto init
|
25
|
-
|
26
|
-
echo.
|
27
|
-
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
-
echo.
|
29
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
-
echo location of your Java installation.
|
31
|
-
|
32
|
-
goto fail
|
33
|
-
|
34
|
-
:findJavaFromJavaHome
|
35
|
-
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
-
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
-
|
38
|
-
if exist "%JAVA_EXE%" goto init
|
39
|
-
|
40
|
-
echo.
|
41
|
-
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
-
echo.
|
43
|
-
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
-
echo location of your Java installation.
|
45
|
-
|
46
|
-
goto fail
|
47
|
-
|
48
|
-
:init
|
49
|
-
@rem Get command-line arguments, handling
|
50
|
-
|
51
|
-
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
-
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
-
|
54
|
-
:win9xME_args
|
55
|
-
@rem Slurp the command line arguments.
|
56
|
-
set CMD_LINE_ARGS=
|
57
|
-
set _SKIP=2
|
58
|
-
|
59
|
-
:win9xME_args_slurp
|
60
|
-
if "x%~1" == "x" goto execute
|
61
|
-
|
62
|
-
set CMD_LINE_ARGS=%*
|
63
|
-
goto execute
|
64
|
-
|
65
|
-
:4NT_args
|
66
|
-
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
-
set CMD_LINE_ARGS=%$
|
68
|
-
|
69
|
-
:execute
|
70
|
-
@rem Setup the command line
|
71
|
-
|
72
|
-
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
-
|
74
|
-
@rem Execute Gradle
|
75
|
-
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
-
|
77
|
-
:end
|
78
|
-
@rem End local scope for the variables with windows NT shell
|
79
|
-
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
-
|
81
|
-
:fail
|
82
|
-
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
-
rem the _cmd.exe /c_ return code!
|
84
|
-
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
-
exit /b 1
|
86
|
-
|
87
|
-
:mainEnd
|
88
|
-
if "%OS%"=="Windows_NT" endlocal
|
89
|
-
|
90
|
-
:omega
|
1
|
+
@if "%DEBUG%" == "" @echo off
|
2
|
+
@rem ##########################################################################
|
3
|
+
@rem
|
4
|
+
@rem Gradle startup script for Windows
|
5
|
+
@rem
|
6
|
+
@rem ##########################################################################
|
7
|
+
|
8
|
+
@rem Set local scope for the variables with windows NT shell
|
9
|
+
if "%OS%"=="Windows_NT" setlocal
|
10
|
+
|
11
|
+
set DIRNAME=%~dp0
|
12
|
+
if "%DIRNAME%" == "" set DIRNAME=.
|
13
|
+
set APP_BASE_NAME=%~n0
|
14
|
+
set APP_HOME=%DIRNAME%
|
15
|
+
|
16
|
+
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
|
17
|
+
set DEFAULT_JVM_OPTS=
|
18
|
+
|
19
|
+
@rem Find java.exe
|
20
|
+
if defined JAVA_HOME goto findJavaFromJavaHome
|
21
|
+
|
22
|
+
set JAVA_EXE=java.exe
|
23
|
+
%JAVA_EXE% -version >NUL 2>&1
|
24
|
+
if "%ERRORLEVEL%" == "0" goto init
|
25
|
+
|
26
|
+
echo.
|
27
|
+
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
|
28
|
+
echo.
|
29
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
30
|
+
echo location of your Java installation.
|
31
|
+
|
32
|
+
goto fail
|
33
|
+
|
34
|
+
:findJavaFromJavaHome
|
35
|
+
set JAVA_HOME=%JAVA_HOME:"=%
|
36
|
+
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
|
37
|
+
|
38
|
+
if exist "%JAVA_EXE%" goto init
|
39
|
+
|
40
|
+
echo.
|
41
|
+
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
|
42
|
+
echo.
|
43
|
+
echo Please set the JAVA_HOME variable in your environment to match the
|
44
|
+
echo location of your Java installation.
|
45
|
+
|
46
|
+
goto fail
|
47
|
+
|
48
|
+
:init
|
49
|
+
@rem Get command-line arguments, handling Windows variants
|
50
|
+
|
51
|
+
if not "%OS%" == "Windows_NT" goto win9xME_args
|
52
|
+
if "%@eval[2+2]" == "4" goto 4NT_args
|
53
|
+
|
54
|
+
:win9xME_args
|
55
|
+
@rem Slurp the command line arguments.
|
56
|
+
set CMD_LINE_ARGS=
|
57
|
+
set _SKIP=2
|
58
|
+
|
59
|
+
:win9xME_args_slurp
|
60
|
+
if "x%~1" == "x" goto execute
|
61
|
+
|
62
|
+
set CMD_LINE_ARGS=%*
|
63
|
+
goto execute
|
64
|
+
|
65
|
+
:4NT_args
|
66
|
+
@rem Get arguments from the 4NT Shell from JP Software
|
67
|
+
set CMD_LINE_ARGS=%$
|
68
|
+
|
69
|
+
:execute
|
70
|
+
@rem Setup the command line
|
71
|
+
|
72
|
+
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
|
73
|
+
|
74
|
+
@rem Execute Gradle
|
75
|
+
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
|
76
|
+
|
77
|
+
:end
|
78
|
+
@rem End local scope for the variables with windows NT shell
|
79
|
+
if "%ERRORLEVEL%"=="0" goto mainEnd
|
80
|
+
|
81
|
+
:fail
|
82
|
+
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
|
83
|
+
rem the _cmd.exe /c_ return code!
|
84
|
+
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
|
85
|
+
exit /b 1
|
86
|
+
|
87
|
+
:mainEnd
|
88
|
+
if "%OS%"=="Windows_NT" endlocal
|
89
|
+
|
90
|
+
:omega
|
@@ -40,44 +40,48 @@ public class RemoteFileInputPlugin
|
|
40
40
|
extends Task {
|
41
41
|
@Config("hosts")
|
42
42
|
@ConfigDefault("[]")
|
43
|
-
|
43
|
+
List<String> getHosts();
|
44
44
|
|
45
45
|
@Config("hosts_command")
|
46
46
|
@ConfigDefault("null")
|
47
|
-
|
47
|
+
Optional<String> getHostsCommand();
|
48
48
|
|
49
49
|
@Config("hosts_separator")
|
50
50
|
@ConfigDefault("\" \"")
|
51
|
-
|
51
|
+
String getHostsSeparator();
|
52
|
+
|
53
|
+
@Config("port")
|
54
|
+
@ConfigDefault("22")
|
55
|
+
int getPort();
|
52
56
|
|
53
57
|
@Config("path")
|
54
58
|
@ConfigDefault("\"\"")
|
55
|
-
|
59
|
+
String getPath();
|
56
60
|
|
57
61
|
@Config("path_command")
|
58
62
|
@ConfigDefault("null")
|
59
|
-
|
63
|
+
Optional<String> getPathCommand();
|
60
64
|
|
61
65
|
@Config("auth")
|
62
66
|
@ConfigDefault("{}")
|
63
|
-
|
67
|
+
Map<String, String> getAuth();
|
64
68
|
|
65
69
|
@Config("ignore_not_found_hosts")
|
66
70
|
@ConfigDefault("false")
|
67
|
-
|
71
|
+
boolean getIgnoreNotFoundHosts();
|
68
72
|
|
69
73
|
@Config("last_target")
|
70
74
|
@ConfigDefault("null")
|
71
|
-
|
75
|
+
Optional<Target> getLastTarget();
|
72
76
|
|
73
|
-
|
77
|
+
void setLastTarget(Optional<Target> lastTarget);
|
74
78
|
|
75
|
-
|
79
|
+
List<Target> getTargets();
|
76
80
|
|
77
|
-
|
81
|
+
void setTargets(List<Target> targets);
|
78
82
|
|
79
83
|
@ConfigInject
|
80
|
-
|
84
|
+
BufferAllocator getBufferAllocator();
|
81
85
|
}
|
82
86
|
|
83
87
|
private final Logger log = Exec.getLogger(getClass());
|
@@ -219,7 +223,7 @@ public class RemoteFileInputPlugin
|
|
219
223
|
|
220
224
|
private boolean exists(Target target, PluginTask task) throws IOException {
|
221
225
|
try (SSHClient client = SSHClient.getInstance()) {
|
222
|
-
client.connect(target.getHost(), task.getAuth());
|
226
|
+
client.connect(target.getHost(), task.getPort(), task.getAuth());
|
223
227
|
|
224
228
|
final String checkCmd = "ls " + target.getPath(); // TODO: windows
|
225
229
|
final int timeout = 5/* second */;
|
@@ -236,7 +240,7 @@ public class RemoteFileInputPlugin
|
|
236
240
|
|
237
241
|
private InputStream download(Target target, PluginTask task) throws IOException {
|
238
242
|
try (SSHClient client = SSHClient.getInstance()) {
|
239
|
-
client.connect(target.getHost(), task.getAuth());
|
243
|
+
client.connect(target.getHost(), task.getPort(), task.getAuth());
|
240
244
|
final ByteArrayOutputStream stream = new ByteArrayOutputStream();
|
241
245
|
client.scpDownload(target.getPath(), stream);
|
242
246
|
return new ByteArrayInputStream(stream.toByteArray());
|
@@ -14,6 +14,7 @@ import java.io.Closeable;
|
|
14
14
|
import java.io.IOException;
|
15
15
|
import java.io.InputStream;
|
16
16
|
import java.io.OutputStream;
|
17
|
+
import java.util.Arrays;
|
17
18
|
import java.util.Map;
|
18
19
|
import java.util.concurrent.TimeUnit;
|
19
20
|
|
@@ -25,27 +26,26 @@ public class SSHClient implements Closeable {
|
|
25
26
|
return new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig(){
|
26
27
|
@Override
|
27
28
|
protected void initSignatureFactories() {
|
28
|
-
setSignatureFactories(
|
29
|
+
setSignatureFactories(Arrays.asList(
|
29
30
|
new SignatureRSA.Factory(),
|
30
31
|
new SignatureECDSA.Factory(),
|
31
32
|
new SignatureDSA.Factory(),
|
32
33
|
new SignatureEdDSA.Factory()
|
33
|
-
);
|
34
|
+
));
|
34
35
|
}
|
35
36
|
}));
|
36
37
|
}
|
37
38
|
|
38
|
-
|
39
|
-
SSHClient(net.schmizz.sshj.SSHClient client) {
|
39
|
+
private SSHClient(net.schmizz.sshj.SSHClient client) {
|
40
40
|
this.client = client;
|
41
41
|
}
|
42
42
|
|
43
|
-
public void connect(String host, Map<String, String> authConfig) throws IOException {
|
43
|
+
public void connect(String host, int port, Map<String, String> authConfig) throws IOException {
|
44
44
|
if (Boolean.valueOf(authConfig.get("skip_host_key_verification"))) {
|
45
45
|
client.addHostKeyVerifier(new PromiscuousVerifier());
|
46
46
|
}
|
47
47
|
client.loadKnownHosts();
|
48
|
-
client.connect(host);
|
48
|
+
client.connect(host, port);
|
49
49
|
|
50
50
|
final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";
|
51
51
|
final String user = authConfig.get("user") != null ? authConfig.get("user") : System.getProperty("user.name");
|
@@ -1,34 +1,38 @@
|
|
1
1
|
package org.embulk.input;
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
3
|
+
import org.embulk.config.ConfigSource;
|
4
|
+
import org.embulk.spi.InputPlugin;
|
5
|
+
import org.embulk.test.EmbulkTests;
|
6
|
+
import org.embulk.test.TestingEmbulk;
|
7
|
+
import org.junit.Rule;
|
8
|
+
import org.junit.Test;
|
9
|
+
|
10
|
+
import java.nio.file.Path;
|
11
|
+
|
12
|
+
import static org.embulk.test.EmbulkTests.readResource;
|
13
|
+
import static org.embulk.test.EmbulkTests.readSortedFile;
|
14
|
+
import static org.hamcrest.Matchers.is;
|
15
|
+
import static org.junit.Assert.assertThat;
|
14
16
|
|
15
17
|
public class TestRemoteFileInputPlugin
|
16
18
|
{
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
19
|
+
@Rule
|
20
|
+
public TestingEmbulk embulk = TestingEmbulk
|
21
|
+
.builder()
|
22
|
+
.registerPlugin(InputPlugin.class, "remote", RemoteFileInputPlugin.class)
|
23
|
+
.build();
|
24
|
+
|
25
|
+
@Test
|
26
|
+
public void loadFromRemote() throws Exception
|
27
|
+
{
|
28
|
+
ConfigSource config = EmbulkTests.config("YAML_TEST01");
|
29
|
+
Path out = embulk.createTempFile("csv");
|
30
|
+
|
31
|
+
embulk.runInput(config, out);
|
32
|
+
|
33
|
+
assertThat(
|
34
|
+
readSortedFile(out),
|
35
|
+
is(readResource("expect/test01.csv")));
|
36
|
+
}
|
37
|
+
|
34
38
|
}
|
@@ -0,0 +1,15 @@
|
|
1
|
+
type: remote
|
2
|
+
hosts:
|
3
|
+
- localhost
|
4
|
+
port: 10022
|
5
|
+
path: /mount/test01.csv
|
6
|
+
auth:
|
7
|
+
user: root
|
8
|
+
type: password
|
9
|
+
password: screencast
|
10
|
+
skip_host_key_verification: true
|
11
|
+
parser:
|
12
|
+
type: csv
|
13
|
+
columns:
|
14
|
+
- {name: id, type: long}
|
15
|
+
- {name: username, type: string}
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-remote
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.10
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shinichi Ishimura
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-12-
|
11
|
+
date: 2016-12-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -46,6 +46,8 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
50
|
+
- Dockerfile
|
49
51
|
- LICENSE.txt
|
50
52
|
- README.md
|
51
53
|
- build.gradle
|
@@ -59,10 +61,13 @@ files:
|
|
59
61
|
- src/main/java/org/embulk/input/RemoteFileInputPlugin.java
|
60
62
|
- src/main/java/org/embulk/input/remote/SSHClient.java
|
61
63
|
- src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
|
64
|
+
- src/test/resources/expect/test01.csv
|
65
|
+
- src/test/resources/input/test01.csv
|
66
|
+
- src/test/resources/yaml/test01.yml
|
62
67
|
- classpath/bcpkix-jdk15on-1.51.jar
|
63
68
|
- classpath/bcprov-jdk15on-1.51.jar
|
64
69
|
- classpath/ecc-25519-java-1.0.1.jar
|
65
|
-
- classpath/embulk-input-remote-0.1.
|
70
|
+
- classpath/embulk-input-remote-0.1.10.jar
|
66
71
|
- classpath/jzlib-1.1.3.jar
|
67
72
|
- classpath/sshj-0.15.0.jar
|
68
73
|
homepage: https://github.com/kamatama41/embulk-input-remote
|
Binary file
|