embulk-input-remote 0.1.10 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 05d25bac2bc3ef17ff27004d9fce3e6317b8be80
-  data.tar.gz: 502556efd3275e388fc7ebb072dacc9bfa524231
+  metadata.gz: ea3bfc2c0371ae959be124623397149f674a3228
+  data.tar.gz: 7b80c2e4233a95201f1481991f7a536ba1bebf9c
 SHA512:
-  metadata.gz: 8a99a040b1900a2458787b658699ea6b449b5c2d95de42b23554b44a152a88f2ef73efa2a3d53299e2a31e51d3147b187166905901c75e78c2787dfc767c920d
-  data.tar.gz: f7203448e89983ca048cd4c85dbc8a503dbbda92bf49f793f5f46ce81c1d4291a3d2e86f0a693e84f515d7731a51a64fe94b57ff71c978c6bc271736251f8733
+  metadata.gz: c635b206f4fdfe0f6e794bada76a2f900ad47f7cd4cbc17f2b6797d6661d5d4f8c279a551c2a20115358b4aa5162683f618504d8a4d7bef30ee7b72a6591ec7c
+  data.tar.gz: b70b926b21825bc5563634e36a77c9155110fd48f56b5f18f28a84553eb101ac8fb73d4bdf6f279d6144577e2257299e1dc41364bf9ce3cc097acc8f5ddf5f81

data/.travis.yml

@@ -6,15 +6,16 @@ jdk:
   - oraclejdk8
 
 env:
-  - YAML_TEST01=$PWD/src/test/resources/yaml/test01.yml
+  - >-
+   KEY_PATH=$PWD/id_rsa_test
 
 services:
   - docker
 
 before_install:
-  - docker build -t eg_sshd .
-  - docker run -d -p 10022:22 -v $PWD/src/test/resources/input:/mount --name test_sshd eg_sshd
-  - docker ps -a
+  - ssh-keygen -t ecdsa -f ${KEY_PATH} -N ''
+  - docker-compose up -d
+  - docker-compose ps
 
 script:
   - ./gradlew --info check

data/Dockerfile

@@ -1,16 +1,17 @@
-# ref: https://docs.docker.com/engine/examples/running_ssh_service/
-FROM ubuntu:16.04
-
-RUN apt-get update && apt-get install -y openssh-server
-RUN mkdir /var/run/sshd
-RUN echo 'root:screencast' | chpasswd
-RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
-
-# SSH login fix. Otherwise user is kicked off after login
-RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
-
-ENV NOTVISIBLE "in users profile"
-RUN echo "export VISIBLE=now" >> /etc/profile
-
-EXPOSE 22
-CMD ["/usr/sbin/sshd", "-D"]
+# Deprecated
+## ref: https://docs.docker.com/engine/examples/running_ssh_service/
+#FROM ubuntu:16.04
+#
+#RUN apt-get update && apt-get install -y openssh-server
+#RUN mkdir /var/run/sshd
+#RUN echo 'root:screencast' | chpasswd
+#RUN sed -i 's/PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config
+#
+## SSH login fix. Otherwise user is kicked off after login
+#RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd
+#
+#ENV NOTVISIBLE "in users profile"
+#RUN echo "export VISIBLE=now" >> /etc/profile
+#
+#EXPOSE 22
+#CMD ["/usr/sbin/sshd", "-D"]

data/README.md

@@ -1,4 +1,4 @@
-[![Build Status](https://travis-ci.org/kamatama41/tfenv.svg?branch=master)](https://travis-ci.org/kamatama41/tfenv)
+[![Build Status](https://travis-ci.org/kamatama41/embulk-input-remote.svg?branch=master)](https://travis-ci.org/kamatama41/embulk-input-remote)
 
 # Remote file input plugin for [Embulk](https://github.com/embulk/embulk)
 
@@ -12,10 +12,10 @@ This plugin load data from Remote hosts by SCP
 
 ## Configuration
 
-- **hosts**: Target hosts (list, default: [])
+- **hosts**: Target hosts, format should be `host` or `host:port` (overrides `default_port`) (list, default: [])
 - **hosts_command**: Command for getting hosts(Windows not supported). If given the option, "hosts" is overwritten. (string, default: null)
 - **hosts_separator**: Separator for "hosts_command" result (string, default: " ")
-- **port**: Port number for SSH (integer, default: 22)
+- **default_port**: Default port number for SSH (integer, default: 22)
 - **path**: Path of remote host (File or Directory) (string, default: "")
 - **path_command**: Command for getting path (Windows not supported). If given the option "path" is overwritten. (string, default: null)
 - **ignore_not_found_hosts**: If the option is true, Hosts which meet the following conditions are skipped. (Means they are not included into resume target.) (boolean, default: false)
@@ -35,7 +35,7 @@ in:
   type: remote
   hosts:
     - host1
-    - host2
+    - host2:10022
 #  hosts_command: echo 'host1,host2'
 #  hosts_separator: ','
   path: /some/path/20150414125923
@@ -50,7 +50,7 @@ in:
 ```
 
 ## Note
-When this plugin run on Linux, task is sometimes blocked.  
+When this plugin run on Linux, a task might be blocked.  
 The cause is java.security.SecureRandom. Please try one of the followings.
 
 ### set JVM_OPTION "-Djava.security.egd"
@@ -71,6 +71,18 @@ securerandom.source=file:/dev/./urandom # after
 
 http://stackoverflow.com/questions/137212/how-to-solve-performance-problem-with-java-securerandom
 
+## Development on local machine
+- Install Docker and then we can create SSH-able containers
+```sh
+$ ssh-keygen -t ecdsa -f ./id_rsa_test -N ''
+$ docker-compose up -d
+$ docker-compose ps
+          Name                 Command       State           Ports         
+--------------------------------------------------------------------------
+embulkinputremote_host1_1   /entrypoint.sh   Up      0.0.0.0:10022->22/tcp 
+embulkinputremote_host2_1   /entrypoint.sh   Up      0.0.0.0:10023->22/tcp 
+```
+
 ## Build
 
 ```

data/build.gradle

@@ -18,15 +18,16 @@ configurations {
     provided
 }
 
-version = "0.1.10"
+version = "0.2.0"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.8.15"
-    compile  "com.hierynomus:sshj:0.15.0"
+    compile  "com.hierynomus:sshj:0.19.1"
     compile  "com.jcraft:jzlib:1.1.3"
     provided "org.embulk:embulk-core:0.8.15"
     testCompile "org.embulk:embulk-standards:0.8.15"
     testCompile "org.embulk:embulk-test:0.8.15"
+    testCompile "com.github.docker-java:docker-java:3.0.7"
 // for local testing
 //    testCompile files('../embulk/embulk-test/build/libs/embulk-test-0.8.15.jar')
 //    testCompile 'junit:junit:4.12'

data/docker-compose.yml

@@ -0,0 +1,16 @@
+version: '2'
+services:
+  host1:
+    image: "sickp/alpine-sshd:7.2"
+    ports:
+     - "10022:22"
+    volumes:
+     - ./src/test/resources/input/host1/:/mount
+     - ./id_rsa_test.pub:/root/.ssh/authorized_keys
+  host2:
+    image: "sickp/alpine-sshd:7.2"
+    ports:
+     - "10023:22"
+    volumes:
+     - ./src/test/resources/input/host2/:/mount
+     - ./id_rsa_test.pub:/root/.ssh/authorized_keys

data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java

@@ -8,9 +8,8 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
-import java.util.Map;
+import java.util.Objects;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -32,8 +31,6 @@ import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamTransactionalFileInput;
 import org.slf4j.Logger;
 
-import javax.annotation.Nullable;
-
 public class RemoteFileInputPlugin
 		implements FileInputPlugin {
 	public interface PluginTask
@@ -50,9 +47,9 @@ public class RemoteFileInputPlugin
 		@ConfigDefault("\" \"")
 		String getHostsSeparator();
 
-		@Config("port")
+		@Config("default_port")
 		@ConfigDefault("22")
-		int getPort();
+		int getDefaultPort();
 
 		@Config("path")
 		@ConfigDefault("\"\"")
@@ -63,18 +60,17 @@ public class RemoteFileInputPlugin
 		Optional<String> getPathCommand();
 
 		@Config("auth")
-		@ConfigDefault("{}")
-		Map<String, String> getAuth();
+		AuthConfig getAuthConfig();
 
 		@Config("ignore_not_found_hosts")
 		@ConfigDefault("false")
 		boolean getIgnoreNotFoundHosts();
 
-		@Config("last_target")
-		@ConfigDefault("null")
-		Optional<Target> getLastTarget();
+		@Config("done_targets")
+		@ConfigDefault("[]")
+		List<Target> getDoneTargets();
 
-		void setLastTarget(Optional<Target> lastTarget);
+		void setDoneTargets(List<Target> lastTarget);
 
 		List<Target> getTargets();
 
@@ -84,6 +80,28 @@ public class RemoteFileInputPlugin
 		BufferAllocator getBufferAllocator();
 	}
 
+	public interface AuthConfig extends Task {
+		@Config("type")
+		@ConfigDefault("\"public_key\"")
+		String getType();
+
+		@Config("user")
+		@ConfigDefault("null")
+		Optional<String> getUser();
+
+		@Config("key_path")
+		@ConfigDefault("null")
+		Optional<String> getKeyPath();
+
+		@Config("password")
+		@ConfigDefault("null")
+		Optional<String> getPassword();
+
+		@Config("skip_host_key_verification")
+		@ConfigDefault("false")
+		boolean getSkipHostKeyVerification();
+	}
+
 	private final Logger log = Exec.getLogger(getClass());
 
 	@Override
@@ -103,11 +121,17 @@ public class RemoteFileInputPlugin
 		final String path = getPath(task);
 
 		final ImmutableList.Builder<Target> builder = ImmutableList.builder();
-		Target lastTarget = task.getLastTarget().orNull();
+		List<Target> doneTargets = task.getDoneTargets();
 		for (String host : hosts) {
-			Target target = new Target(host, path);
+			String[] split = host.split(":");
+			final String targetHost = split[0];
+			int targetPort = task.getDefaultPort();
+			if (split.length > 1) {
+				targetPort = Integer.valueOf(split[1]);
+			}
+			Target target = new Target(targetHost, targetPort, path);
 
-			if (lastTarget == null || target.compareTo(lastTarget) > 0) {
+			if (!doneTargets.contains(target)) {
 				if (task.getIgnoreNotFoundHosts()) {
 					try {
 						final boolean exists = exists(target, task);
@@ -180,13 +204,8 @@ public class RemoteFileInputPlugin
 		control.run(taskSource, taskCount);
 
 		List<Target> targets = new ArrayList<>(task.getTargets());
-		Collections.sort(targets);
-
-		if (targets.isEmpty()) {
-			return Exec.newConfigDiff();
-		}
 
-		return Exec.newConfigDiff().set("last_target", targets.get(targets.size() - 1));
+		return Exec.newConfigDiff().set("done_targets", targets);
 	}
 
 	@Override
@@ -222,9 +241,7 @@ public class RemoteFileInputPlugin
 	}
 
 	private boolean exists(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = SSHClient.getInstance()) {
-			client.connect(target.getHost(), task.getPort(), task.getAuth());
-
+		try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
 			final String checkCmd = "ls " + target.getPath();    // TODO: windows
 			final int timeout = 5/* second */;
 			final SSHClient.CommandResult commandResult = client.execCommand(checkCmd, timeout);
@@ -239,23 +256,26 @@ public class RemoteFileInputPlugin
 	}
 
 	private InputStream download(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = SSHClient.getInstance()) {
-			client.connect(target.getHost(), task.getPort(), task.getAuth());
+		try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
 			final ByteArrayOutputStream stream = new ByteArrayOutputStream();
 			client.scpDownload(target.getPath(), stream);
 			return new ByteArrayInputStream(stream.toByteArray());
 		}
 	}
 
-	public static class Target implements Comparable<Target> {
+	public static class Target {
 		private final String host;
+		private final int port;
 		private final String path;
 
 		@JsonCreator
 		public Target(
 				@JsonProperty("host") String host,
-				@JsonProperty("path") String path) {
+				@JsonProperty("port") int port,
+				@JsonProperty("path") String path
+		) {
 			this.host = host;
+			this.port = port;
 			this.path = path;
 		}
 
@@ -263,21 +283,32 @@ public class RemoteFileInputPlugin
 			return host;
 		}
 
+		public int getPort() {
+			return port;
+		}
+
 		public String getPath() {
 			return path;
 		}
 
 		@Override
-		public int compareTo(@Nullable Target other) {
-			if (other == null) {
-				throw new NullPointerException();
-			}
-			return this.toString().compareTo(other.toString());
+		public boolean equals(Object o) {
+			if (this == o) return true;
+			if (o == null || getClass() != o.getClass()) return false;
+			Target target = (Target) o;
+			return port == target.port &&
+					Objects.equals(host, target.host) &&
+					Objects.equals(path, target.path);
+		}
+
+		@Override
+		public int hashCode() {
+			return Objects.hash(host, port, path);
 		}
 
 		@Override
 		public String toString() {
-			return host + ":" + path;
+			return host + ":" + port + ":" + path;
 		}
 	}
 }

data/src/main/java/org/embulk/input/remote/SSHClient.java

@@ -1,63 +1,56 @@
 package org.embulk.input.remote;
 
-import com.hierynomus.sshj.signature.SignatureEdDSA;
 import net.schmizz.sshj.DefaultConfig;
 import net.schmizz.sshj.connection.channel.direct.Session;
-import net.schmizz.sshj.signature.SignatureDSA;
-import net.schmizz.sshj.signature.SignatureECDSA;
-import net.schmizz.sshj.signature.SignatureRSA;
 import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
 import net.schmizz.sshj.xfer.InMemoryDestFile;
 import net.schmizz.sshj.xfer.LocalDestFile;
+import org.embulk.input.RemoteFileInputPlugin;
 
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
-import java.util.Arrays;
-import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 public class SSHClient implements Closeable {
 
 	private final net.schmizz.sshj.SSHClient client;
 
-	public static SSHClient getInstance() {
-		return new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig(){
-			@Override
-			protected void initSignatureFactories() {
-				setSignatureFactories(Arrays.asList(
-						new SignatureRSA.Factory(),
-						new SignatureECDSA.Factory(),
-						new SignatureDSA.Factory(),
-						new SignatureEdDSA.Factory()
-				));
-			}
-		}));
+	public static SSHClient connect(
+			String host, int port, RemoteFileInputPlugin.AuthConfig authConfig
+	) throws IOException {
+
+		SSHClient client =  new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig()));
+		client.connectToHost(host, port, authConfig);
+		return client;
 	}
 
 	private SSHClient(net.schmizz.sshj.SSHClient client) {
 		this.client = client;
 	}
 
-	public void connect(String host, int port, Map<String, String> authConfig) throws IOException {
-		if (Boolean.valueOf(authConfig.get("skip_host_key_verification"))) {
+	private void connectToHost(String host, int port, RemoteFileInputPlugin.AuthConfig authConfig) throws IOException {
+		if (authConfig.getSkipHostKeyVerification()) {
 			client.addHostKeyVerifier(new PromiscuousVerifier());
 		}
 		client.loadKnownHosts();
 		client.connect(host, port);
 
-		final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";
-		final String user = authConfig.get("user") != null ? authConfig.get("user") : System.getProperty("user.name");
+		final String type = authConfig.getType();
+		final String user = authConfig.getUser().or(System.getProperty("user.name"));
 
 		if ("password".equals(type)) {
-			client.authPassword(user, authConfig.get("password"));
+			if (authConfig.getPassword().isPresent()) {
+				client.authPassword(user, authConfig.getPassword().get());
+			} else {
+				throw new IllegalStateException("Password is not set.");
+			}
 		} else if ("public_key".equals(type)) {
-			final String key_path = authConfig.get("key_path");
-			if (key_path == null) {
-				client.authPublickey(user);
+			if (authConfig.getKeyPath().isPresent()) {
+				client.authPublickey(user, authConfig.getKeyPath().get());
 			} else {
-				client.authPublickey(user, key_path);
+				client.authPublickey(user);
 			}
 		} else {
 			throw new UnsupportedOperationException("Unsupported auth type : " + type);

data/src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java

@@ -1,38 +1,255 @@
 package org.embulk.input;
 
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.core.DockerClientBuilder;
+import org.embulk.EmbulkEmbed;
 import org.embulk.config.ConfigSource;
 import org.embulk.spi.InputPlugin;
-import org.embulk.test.EmbulkTests;
+import org.embulk.test.MemoryOutputPlugin;
+import org.embulk.test.MyEmbulkTests;
+import org.embulk.test.MyTestingEmbulk;
 import org.embulk.test.TestingEmbulk;
+import org.junit.Before;
+import org.junit.Ignore;
 import org.junit.Rule;
 import org.junit.Test;
+import org.slf4j.LoggerFactory;
 
-import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
 
-import static org.embulk.test.EmbulkTests.readResource;
-import static org.embulk.test.EmbulkTests.readSortedFile;
 import static org.hamcrest.Matchers.is;
 import static org.junit.Assert.assertThat;
 
-public class TestRemoteFileInputPlugin
-{
+public class TestRemoteFileInputPlugin {
+    private static final String CONTAINER_ID_HOST1 = "embulkinputremote_host1_1";
+    private static final String CONTAINER_ID_HOST2 = "embulkinputremote_host2_1";
+    private static final DockerClient dockerClient = DockerClientBuilder.getInstance().build();
+
     @Rule
-    public TestingEmbulk embulk = TestingEmbulk
+    public MyTestingEmbulk embulk = (MyTestingEmbulk) MyTestingEmbulk
             .builder()
             .registerPlugin(InputPlugin.class, "remote", RemoteFileInputPlugin.class)
             .build();
 
+    @Before
+    public void prepare() {
+        startContainer(CONTAINER_ID_HOST1);
+        startContainer(CONTAINER_ID_HOST2);
+
+        String logLevel = System.getenv("LOG_LEVEL");
+        if (logLevel != null) {
+            // Set log level
+            Logger rootLogger = (Logger) LoggerFactory.getLogger(Logger.ROOT_LOGGER_NAME);
+            rootLogger.setLevel(Level.toLevel(logLevel));
+        }
+    }
+
     @Test
     public void loadFromRemote() throws Exception
     {
-        ConfigSource config = EmbulkTests.config("YAML_TEST01");
-        Path out = embulk.createTempFile("csv");
+        embulk.runInput(baseConfig());
+        assertValues(values(1L, "user1"));
+    }
+
+    @Ignore("Cannot pass on TravisCI, although pass on Local Mac OS...")
+    @Test
+    public void loadFromRemoteViaPublicKey() throws Exception
+    {
+        String keyPath = System.getenv("KEY_PATH");
+        if (keyPath == null) {
+            keyPath = "./id_rsa_test";
+        }
+
+        final ConfigSource publicKeyAuth = newConfig().set("auth", newConfig()
+                .set("type", "public_key")
+                .set("key_path", keyPath)
+        );
+        embulk.runInput(baseConfig().merge(publicKeyAuth));
+
+        assertValues(values(1L, "user1"));
+    }
+
+    @Test
+    public void testMultiHosts() throws Exception
+    {
+        final ConfigSource multiHosts = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
+        final ConfigSource config = baseConfig().merge(multiHosts);
+
+        // Run
+        embulk.runInput(config);
+        assertValues(
+                values(1L, "user1"),
+                values(2L, "user2")
+        );
+    }
+
+    @Test
+    public void loadAllFilesInDirectory() throws Exception
+    {
+        final ConfigSource multiHosts = newConfig()
+                .set("path", "/mount");
+        final ConfigSource config = baseConfig().merge(multiHosts);
+
+        embulk.runInput(config);
+        assertValues(
+                values(1L, "user1"),
+                values(1L, "command_user1")
+        );
+    }
+
+    @Test
+    public void testDefaultPort() throws Exception
+    {
+        final ConfigSource defaultPort = newConfig()
+                .set("hosts", Collections.singletonList("localhost"))
+                .set("default_port", 10022);
+
+        embulk.runInput(baseConfig().merge(defaultPort));
+
+        assertValues(values(1L, "user1"));
+    }
+
+    @Test
+    public void testConfDiff() throws Exception
+    {
+        final ConfigSource host2Config = newConfig()
+                .set("hosts", Collections.singletonList("localhost:10023"));
+        ConfigSource config = baseConfig().merge(host2Config);
+
+        // Run
+        TestingEmbulk.RunResult runResult = embulk.runInput(config);
+        assertValues(values(2L, "user2"));
+
+        // Re-run with additional host1
+        final ConfigSource multiHost = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
+        config = baseConfig().merge(multiHost);
 
-        embulk.runInput(config, out);
+        embulk.runInput(config, runResult.getConfigDiff());
 
-        assertThat(
-                readSortedFile(out),
-                is(readResource("expect/test01.csv")));
+        assertValues(values(1L, "user1"));
     }
 
+    @Test
+    public void testResume() throws Exception
+    {
+        final ConfigSource multiHost = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"));
+        final ConfigSource config = baseConfig().merge(multiHost);
+
+        // Stop host2 temporarily
+        stopContainer(CONTAINER_ID_HOST2);
+
+        // Run (but will fail)
+        EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
+
+        assertThat(resumableResult.isSuccessful(), is(false));
+        assertValues(values(1L, "user1"));
+
+        // Start host2 again
+        startContainer(CONTAINER_ID_HOST2);
+
+        // Resume
+        resumableResult = embulk.resume(config, resumableResult.getResumeState());
+
+        assertThat(resumableResult.isSuccessful(), is(true));
+        assertValues(values(2L, "user2"));
+    }
+
+    @Test
+    public void testIgnoreNotFoundHosts() throws Exception
+    {
+        final ConfigSource ignoreNotFoundHosts = newConfig()
+                .set("hosts", Arrays.asList("localhost:10022", "localhost:10023"))
+                .set("ignore_not_found_hosts", true);
+        final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
+
+        // Stop host2
+        stopContainer(CONTAINER_ID_HOST2);
+
+        // Run (host2 will be ignored)
+        EmbulkEmbed.ResumableResult resumableResult = embulk.resume(config);
+
+        assertThat(resumableResult.isSuccessful(), is(true));
+        assertValues(values(1L, "user1"));
+    }
+
+    @Test
+    public void testCommandOptions() throws Exception
+    {
+        final ConfigSource ignoreNotFoundHosts = newConfig()
+                .set("hosts_command", "echo localhost:10022,localhost:10023")
+                .set("hosts_separator", ",")
+                .set("path_command", "echo /mount/test_command.csv");
+        final ConfigSource config = baseConfig().merge(ignoreNotFoundHosts);
+
+        embulk.runInput(config);
+
+        assertValues(
+                values(1L, "command_user1"),
+                values(2L, "command_user2")
+        );
+    }
+
+    private ConfigSource baseConfig() {
+        return MyEmbulkTests.configFromResource("yaml/base.yml");
+    }
+
+    private ConfigSource newConfig() {
+        return embulk.newConfig();
+    }
+
+    private void assertValues(List... valuesList) {
+        Set<List> actual = new HashSet<>();
+        for (MemoryOutputPlugin.Record record : MemoryOutputPlugin.getRecords()) {
+            actual.add(record.getValues());
+        }
+
+        Set<List> expected = new HashSet<>();
+        Collections.addAll(expected, valuesList);
+
+        assertThat(actual, is(expected));
+    }
+
+    private List values(Object... values) {
+        return Arrays.asList(values);
+    }
+
+    //////////////////////////////
+    // Methods for Docker
+    //////////////////////////////
+
+    private static void stopContainer(String containerId) {
+        if (isRunning(containerId)) {
+            dockerClient.stopContainerCmd(containerId).exec();
+        }
+    }
+
+    private static void startContainer(String containerId) {
+        if (!isRunning(containerId)) {
+            dockerClient.startContainerCmd(containerId).exec();
+        }
+    }
+
+    private static boolean isRunning(String containerId) {
+        List<Container> containers = dockerClient.listContainersCmd().exec();
+        for (Container container : containers) {
+            for (String name : container.getNames()) {
+                if (name.contains(containerId)) {
+                    System.out.println("Found " + containerId);
+                    return true;
+                }
+            }
+        }
+        System.out.println("Not Found " + containerId);
+        return false;
+    }
 }

data/src/test/java/org/embulk/test/MemoryOutputPlugin.java

@@ -0,0 +1,143 @@
+package org.embulk.test;
+
+import com.google.common.collect.ImmutableList;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Column;
+import org.embulk.spi.Exec;
+import org.embulk.spi.OutputPlugin;
+import org.embulk.spi.Page;
+import org.embulk.spi.PageReader;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TransactionalPageOutput;
+import org.embulk.spi.util.Pages;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class MemoryOutputPlugin implements OutputPlugin
+{
+    public interface PluginTask extends Task { }
+
+    @Override
+    public ConfigDiff transaction(ConfigSource config,
+                                  Schema schema, int taskCount,
+                                  OutputPlugin.Control control)
+    {
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        return resume(task.dump(), schema, taskCount, control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+                             Schema schema, int taskCount,
+                             OutputPlugin.Control control)
+    {
+        control.run(taskSource);
+        return Exec.newConfigDiff();
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+                        Schema schema, int taskCount,
+                        List<TaskReport> successTaskReports)
+    { }
+
+    @Override
+    public TransactionalPageOutput open(final TaskSource taskSource, final Schema schema, final int taskIndex)
+    {
+        return new TransactionalPageOutput()
+        {
+            private final PageReader reader = new PageReader(schema);
+
+            public void add(Page page)
+            {
+                reader.setPage(page);
+                while (reader.nextRecord())
+                {
+                    Recorder.addRecord(reader);
+                }
+            }
+
+            public void finish() { }
+
+            public void close()
+            {
+                reader.close();
+            }
+
+            public void abort() { }
+
+            public TaskReport commit()
+            {
+                return Exec.newTaskReport();
+            }
+        };
+    }
+
+    public static void clearRecords()
+    {
+        Recorder.clearRecords();
+    }
+
+    public static List<Record> getRecords()
+    {
+        return Recorder.getRecords();
+    }
+
+    private static class Recorder
+    {
+        private static List<Record> records = new ArrayList<>();
+
+        private Recorder() { }
+
+        private synchronized static void addRecord(PageReader reader)
+        {
+            final ImmutableList.Builder<Object> values = ImmutableList.builder();
+            final ImmutableList.Builder<Column> columns = ImmutableList.builder();
+            reader.getSchema().visitColumns(new Pages.ObjectColumnVisitor(reader) {
+                @Override
+                public void visit(org.embulk.spi.Column column, Object value) {
+                    values.add(value);
+                    columns.add(column);
+                }
+            });
+            records.add(new Record(values.build(), columns.build()));
+        }
+
+        static void clearRecords()
+        {
+            records = new ArrayList<>();
+        }
+
+        static List<Record> getRecords()
+        {
+            return new ArrayList<>(records);
+        }
+    }
+
+    public static class Record
+    {
+        private final List<Object> values;
+        private final List<Column> columns;
+
+        Record(List<Object> values, List<Column> columns)
+        {
+            this.values = values;
+            this.columns = columns;
+        }
+
+        public List<Object> getValues()
+        {
+            return values;
+        }
+
+        public List<Column> getColumns()
+        {
+            return columns;
+        }
+    }
+}

data/src/test/java/org/embulk/test/MyEmbulkTests.java

@@ -0,0 +1,23 @@
+package org.embulk.test;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigSource;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+import static org.embulk.test.EmbulkTests.readResource;
+import static org.hamcrest.Matchers.is;
+import static org.junit.Assume.assumeThat;
+
+public class MyEmbulkTests {
+    private MyEmbulkTests() {
+    }
+
+    public static ConfigSource configFromString(String yaml) {
+        assumeThat(isNullOrEmpty(yaml), is(false));
+        return EmbulkEmbed.newSystemConfigLoader().fromYamlString(yaml);
+    }
+
+    public static ConfigSource configFromResource(String name) {
+        return configFromString(readResource(name));
+    }
+}

data/src/test/java/org/embulk/test/MyTestingEmbulk.java

@@ -0,0 +1,92 @@
+package org.embulk.test;
+
+import org.embulk.EmbulkEmbed;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.exec.ResumeState;
+import org.embulk.spi.OutputPlugin;
+
+import java.io.IOException;
+import java.lang.reflect.Field;
+
+public class MyTestingEmbulk extends TestingEmbulk {
+
+    public static class Builder extends TestingEmbulk.Builder {
+        public TestingEmbulk build() {
+            this.registerPlugin(OutputPlugin.class, "memory", MemoryOutputPlugin.class);
+            return new MyTestingEmbulk(this);
+        }
+    }
+
+    public static TestingEmbulk.Builder builder()
+    {
+        return new MyTestingEmbulk.Builder();
+    }
+
+    private final EmbulkEmbed superEmbed;
+
+    MyTestingEmbulk(Builder builder) {
+        super(builder);
+        this.superEmbed = extractSuperField("embed");
+    }
+
+    public RunResult runInput(ConfigSource inConfig) throws IOException {
+        return runInput(inConfig, (ConfigDiff) null);
+    }
+
+    public RunResult runInput(ConfigSource inConfig, ConfigDiff confDiff) throws IOException {
+        ConfigSource execConfig = newConfig()
+                .set("min_output_tasks", 1);
+
+        ConfigSource outConfig = newConfig()
+                .set("type", "memory");
+
+        ConfigSource config = newConfig()
+                .set("exec", execConfig)
+                .set("in", inConfig)
+                .set("out", outConfig);
+
+        // embed.run returns TestingBulkLoader.TestingExecutionResult because
+        MemoryOutputPlugin.clearRecords();
+        if (confDiff == null) {
+            return (RunResult) superEmbed.run(config);
+        } else {
+            return (RunResult) superEmbed.run(config.merge(confDiff));
+        }
+    }
+
+    public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig) throws IOException {
+        return resume(inConfig, null);
+    }
+
+    public EmbulkEmbed.ResumableResult resume(ConfigSource inConfig, ResumeState resumeState) throws IOException {
+        ConfigSource execConfig = newConfig()
+                .set("min_output_tasks", 1);
+
+        ConfigSource outConfig = newConfig()
+                .set("type", "memory");
+
+        ConfigSource config = newConfig()
+                .set("exec", execConfig)
+                .set("in", inConfig)
+                .set("out", outConfig);
+
+        MemoryOutputPlugin.clearRecords();
+        if (resumeState == null) {
+            return superEmbed.runResumable(config);
+        } else {
+            return superEmbed.new ResumeStateAction(config, resumeState).resume();
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private <T> T extractSuperField(String fieldName) {
+        try {
+            Field field = TestingEmbulk.class.getDeclaredField(fieldName);
+            field.setAccessible(true);
+            return (T) field.get(this);
+        } catch (NoSuchFieldException | IllegalAccessException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

data/src/test/resources/input/host1/test.csv

@@ -0,0 +1 @@
+1,user1

data/src/test/resources/input/host1/test_command.csv

@@ -0,0 +1 @@
+1,command_user1

data/src/test/resources/input/host2/test.csv

@@ -0,0 +1 @@
+2,user2

data/src/test/resources/input/host2/test_command.csv

@@ -0,0 +1 @@
+2,command_user2

data/src/test/resources/yaml/{test01.yml → base.yml}

@@ -1,12 +1,11 @@
 type: remote
 hosts:
-  - localhost
-port: 10022
-path: /mount/test01.csv
+  - localhost:10022
+path: /mount/test.csv
 auth:
   user: root
   type: password
-  password: screencast
+  password: root
   skip_host_key_verification: true
 parser:
   type: csv

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.1.10
+  version: 0.2.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-12-25 00:00:00.000000000 Z
+date: 2017-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -51,6 +51,7 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
+- docker-compose.yml
 - example/csv/sample_01.csv.gz
 - example/example.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
@@ -61,15 +62,20 @@ files:
 - src/main/java/org/embulk/input/RemoteFileInputPlugin.java
 - src/main/java/org/embulk/input/remote/SSHClient.java
 - src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
-- src/test/resources/expect/test01.csv
-- src/test/resources/input/test01.csv
-- src/test/resources/yaml/test01.yml
+- src/test/java/org/embulk/test/MemoryOutputPlugin.java
+- src/test/java/org/embulk/test/MyEmbulkTests.java
+- src/test/java/org/embulk/test/MyTestingEmbulk.java
+- src/test/resources/input/host1/test.csv
+- src/test/resources/input/host1/test_command.csv
+- src/test/resources/input/host2/test.csv
+- src/test/resources/input/host2/test_command.csv
+- src/test/resources/yaml/base.yml
 - classpath/bcpkix-jdk15on-1.51.jar
 - classpath/bcprov-jdk15on-1.51.jar
-- classpath/ecc-25519-java-1.0.1.jar
-- classpath/embulk-input-remote-0.1.10.jar
+- classpath/eddsa-0.1.0.jar
+- classpath/embulk-input-remote-0.2.0.jar
 - classpath/jzlib-1.1.3.jar
-- classpath/sshj-0.15.0.jar
+- classpath/sshj-0.19.1.jar
 homepage: https://github.com/kamatama41/embulk-input-remote
 licenses:
 - MIT

data/src/test/resources/expect/test01.csv

@@ -1,2 +0,0 @@
-1,kamatama41
-2,kamatama42

data/src/test/resources/input/test01.csv

@@ -1,2 +0,0 @@
-1,kamatama41
-2,kamatama42