embulk-input-remote 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b37822e56bd1e3b9726a1cdab656c33e42b4d3cf
-  data.tar.gz: 3e567cc0300e1319bccd86fea0410ec7193fc269
+  metadata.gz: 1ea2dc9f5f71195a19ac5f0a5a3e96d4020a02b2
+  data.tar.gz: 4081a11e980990e00243581ec48016cd6bd15bbe
 SHA512:
-  metadata.gz: 4d2157a0e77f281c030535eaf37ab47f28f4f73b5d85d1749ed7fd56fd2fb2676c6661b53271c67e0912885e88028dbd047f5ae4757036962b222c077a9e80b1
-  data.tar.gz: 87ded4537e43dd1477462998083732f4e2dfe0b274a007380a9c4bec1df7c4020a9a792b8681921e3f552a9d6f311fef772560bb7d0bb44872519d55005116e3
+  metadata.gz: 93cfc49c50e5844ad4e9a9b51ea8513d2e8f422866d8c52542f2b306605a78896f1d805bd68a7e21e77114af525e1c3f5510c98b1b74f94743360a79074eb54a
+  data.tar.gz: 34c3fcb851fa472051aeaa77182bfcf89ff2f8b3ee021681cd3904e53ffb31e55719bb33949ed2326ebfc29a2996c4ad3ccd97117358f8abb0c469df5cd67c37

data/README.md

@@ -1,4 +1,4 @@
-# Remote file input plugin for Embulk
+# Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
 
 This plugin load data from Remote hosts by SCP
 
@@ -15,6 +15,7 @@ This plugin load data from Remote hosts by SCP
 - **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
 - **path**: Path of remote host (File or Directory) (string, default: "")
 - **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
+- **ignore_not_found_hosts**: If the option is true, Hosts which file(or directory) is not found is skipped. (Means it's not included in resume target.) (boolean, default: false)
 - **auth**: SSH authentication setting (hash, default: {})
     - **user**: SSH username (string, default: execute user)
     - **type**: public_key or password (string, default: public_key)
@@ -33,6 +34,7 @@ in:
 #  hosts_separator: ','
   path: /some/path/20150414125923
 #  path_command: echo /some/path/`date "+%Y%m%d%H%M%S"`
+  ignore_not_found_hosts: true
   auth:
     user: {username}
     type: public_key

data/build.gradle

@@ -13,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.6.1"

data/gradlew.bat

@@ -1,90 +1,90 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java

@@ -6,7 +6,6 @@ import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.InputStreamReader;
-import java.io.OutputStream;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -17,9 +16,6 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-import net.schmizz.sshj.SSHClient;
-import net.schmizz.sshj.xfer.InMemoryDestFile;
-import net.schmizz.sshj.xfer.LocalDestFile;
 import org.embulk.config.CommitReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
@@ -28,6 +24,7 @@ import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
 import org.embulk.config.TaskSource;
+import org.embulk.input.remote.SSHClient;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
@@ -65,6 +62,10 @@ public class RemoteFileInputPlugin
 		@ConfigDefault("{}")
 		public Map<String, String> getAuth();
 
+		@Config("ignore_not_found_hosts")
+		@ConfigDefault("false")
+		public boolean getIgnoreNotFoundHosts();
+
 		@Config("last_target")
 		@ConfigDefault("null")
 		public Optional<Target> getLastTarget();
@@ -84,19 +85,21 @@ public class RemoteFileInputPlugin
 	@Override
 	public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
 		PluginTask task = config.loadConfig(PluginTask.class);
+		try {
+			List<Target> targets = listTargets(task);
+			log.info("Loading targets {}", targets);
+			task.setTargets(targets);
+	
+			// number of processors is same with number of targets
+			int taskCount = targets.size();
+			return resume(task.dump(), taskCount, control);
 
-		List<Target> targets = listTargets(task);
-		log.info("Loading targets {}", targets);
-		task.setTargets(targets);
-
-		// number of processors is same with number of targets
-		int taskCount = targets.size();
-		return resume(task.dump(), taskCount, control);
+		} catch (IOException e) {
+			throw new RuntimeException(e);
+		}
 	}
 
-	private List<Target> listTargets(PluginTask task) {
-		task.getHostsCommand().orNull();
-
+	private List<Target> listTargets(PluginTask task) throws IOException {
 		final List<String> hosts = listHosts(task);
 		final String path = getPath(task);
 
@@ -104,7 +107,15 @@ public class RemoteFileInputPlugin
 		Target lastTarget = task.getLastTarget().orNull();
 		for (String host : hosts) {
 			Target target = new Target(host, path);
+
 			if (lastTarget == null || target.compareTo(lastTarget) > 0) {
+				if (task.getIgnoreNotFoundHosts()) {
+					// Check with file existing
+					if (!exists(target, task)) {
+						continue;
+					}
+					// This host will fail when "open" method is called.
+				}
 				builder.add(target);
 			}
 		}
@@ -182,7 +193,7 @@ public class RemoteFileInputPlugin
 		final Target target = task.getTargets().get(taskIndex);
 
 		try {
-			return new PluginFileInput(task, download(target, task.getAuth()));
+			return new PluginFileInput(task, download(target, task));
 		} catch (IOException e) {
 			throw new RuntimeException(e);
 		}
@@ -227,55 +238,31 @@ public class RemoteFileInputPlugin
 		public CommitReport commit() {
 			return Exec.newCommitReport();
 		}
+	}
 
+	private boolean exists(Target target, PluginTask task) throws IOException {
+		try (SSHClient client = new SSHClient()) {
+			client.connect(target.getHost(), task.getAuth());
 
-	}
+			final String checkCmd = "ls " + target.getPath();    // TODO: windows
+			final int timeout = 5/* second */;
+			final SSHClient.CommandResult commandResult = client.execCommand(checkCmd, timeout);
 
-	private InputStream download(Target target, Map<String, String> auth) throws IOException {
-		SSHClient client = new SSHClient();
-		client.loadKnownHosts();
-		InMemoryDestFileImpl memoryDestFile = new InMemoryDestFileImpl();
-		try {
-			client.connect(target.getHost());
-			final String type = auth.get("type") != null ? auth.get("type") : "public_key";
-			final String user = auth.get("user") != null ? auth.get("user") : System.getProperty("user.name");
-
-			if ("password".equals(type)) {
-				client.authPassword(user, auth.get("password"));
-			} else if ("public_key".equals(type)) {
-				final String key_path = auth.get("key_path");
-				if (key_path == null) {
-					client.authPublickey(user);
-				} else {
-					client.authPublickey(user, key_path);
-				}
+			if(commandResult.getStatus() != 0) {
+				log.warn("Remote file not found. {}", target.toString());
+				return false;
 			} else {
-				throw new UnsupportedOperationException("Unsupported auth type : " + type);
+				return true;
 			}
-			client.newSCPFileTransfer().download(target.getPath(), memoryDestFile);
-			return new ByteArrayInputStream(memoryDestFile.outputStream.toByteArray());
-
-		} finally {
-			client.disconnect();
 		}
 	}
 
-	public static class InMemoryDestFileImpl extends InMemoryDestFile {
-
-		private ByteArrayOutputStream outputStream;
-
-		public InMemoryDestFileImpl() {
-			this.outputStream = new ByteArrayOutputStream();
-		}
-
-		@Override
-		public OutputStream getOutputStream() throws IOException {
-			return outputStream;
-		}
-
-		@Override
-		public LocalDestFile getTargetDirectory(String dirname) throws IOException {
-			return this;
+	private InputStream download(Target target, PluginTask task) throws IOException {
+		try (SSHClient client = new SSHClient()) {
+			client.connect(target.getHost(), task.getAuth());
+			final ByteArrayOutputStream stream = new ByteArrayOutputStream();
+			client.scpDownload(target.getPath(), stream);
+			return new ByteArrayInputStream(stream.toByteArray());
 		}
 	}
 

data/src/main/java/org/embulk/input/remote/SSHClient.java

@@ -0,0 +1,104 @@
+package org.embulk.input.remote;
+
+import net.schmizz.sshj.connection.channel.direct.Session;
+import net.schmizz.sshj.xfer.InMemoryDestFile;
+import net.schmizz.sshj.xfer.LocalDestFile;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+public class SSHClient implements Closeable {
+
+	private final net.schmizz.sshj.SSHClient client;
+
+	public SSHClient() {
+		this(new net.schmizz.sshj.SSHClient());
+	}
+
+	/* package for test */
+	SSHClient(net.schmizz.sshj.SSHClient client) {
+		this.client = client;
+	}
+
+	public void connect(String host, Map<String, String> authConfig) throws IOException {
+		client.loadKnownHosts();
+
+		client.connect(host);
+
+		final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";
+		final String user = authConfig.get("user") != null ? authConfig.get("user") : System.getProperty("user.name");
+
+		if ("password".equals(type)) {
+			client.authPassword(user, authConfig.get("password"));
+		} else if ("public_key".equals(type)) {
+			final String key_path = authConfig.get("key_path");
+			if (key_path == null) {
+				client.authPublickey(user);
+			} else {
+				client.authPublickey(user, key_path);
+			}
+		} else {
+			throw new UnsupportedOperationException("Unsupported auth type : " + type);
+		}
+	}
+
+	public CommandResult execCommand(String command, int timeoutSecond) throws IOException {
+		try (final Session session = client.startSession()) {
+			final Session.Command cmd = session.exec(command);
+			cmd.join(timeoutSecond, TimeUnit.SECONDS);
+			return new CommandResult(cmd.getExitStatus(), cmd.getInputStream());
+		}
+	}
+
+	public void scpDownload(String path, OutputStream stream) throws IOException {
+		client.newSCPFileTransfer().download(path, new InMemoryDestFileImpl(stream));
+	}
+
+	private static class InMemoryDestFileImpl extends InMemoryDestFile {
+
+		private OutputStream outputStream;
+
+		public InMemoryDestFileImpl(OutputStream outputStream) {
+			this.outputStream = outputStream;
+		}
+
+		@Override
+		public OutputStream getOutputStream() throws IOException {
+			return outputStream;
+		}
+
+		@Override
+		public LocalDestFile getTargetDirectory(String dirname) throws IOException {
+			return this;
+		}
+	}
+
+	@Override
+	public void close() throws IOException {
+		if (client != null) {
+			client.close();
+		}
+	}
+
+	public static class CommandResult {
+		int status;
+		InputStream stdout;
+
+		private CommandResult(int status, InputStream stdout) {
+			this.status = status;
+			this.stdout = stdout;
+		}
+
+		public int getStatus() {
+			return status;
+		}
+
+		public InputStream getStdout() {
+			return stdout;
+		}
+	}
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Shinichi ISHIMURA
@@ -55,10 +55,11 @@ files:
 - gradlew.bat
 - lib/embulk/input/remote.rb
 - src/main/java/org/embulk/input/RemoteFileInputPlugin.java
+- src/main/java/org/embulk/input/remote/SSHClient.java
 - src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
 - classpath/bcpkix-jdk15on-1.51.jar
 - classpath/bcprov-jdk15on-1.51.jar
-- classpath/embulk-input-remote-0.1.1.jar
+- classpath/embulk-input-remote-0.1.2.jar
 - classpath/jzlib-1.1.3.jar
 - classpath/sshj-0.11.0.jar
 homepage: https://github.com/kamatama41/embulk-input-remote