embulk-input-remote 0.2.0 → 0.3.0

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-22 00:00:00.000000000 Z
+date: 2017-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,29 +52,28 @@ files:
 - README.md
 - build.gradle
 - docker-compose.yml
-- example/csv/sample_01.csv.gz
-- example/example.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
 - gradlew.bat
 - lib/embulk/input/remote.rb
-- src/main/java/org/embulk/input/RemoteFileInputPlugin.java
-- src/main/java/org/embulk/input/remote/SSHClient.java
-- src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
-- src/test/java/org/embulk/test/MemoryOutputPlugin.java
-- src/test/java/org/embulk/test/MyEmbulkTests.java
-- src/test/java/org/embulk/test/MyTestingEmbulk.java
+- settings.gradle
+- src/main/kotlin/org/embulk/input/RemoteFileInputPlugin.kt
+- src/main/kotlin/org/embulk/input/remote/SSHClient.kt
+- src/test/kotlin/org/embulk/input/TestRemoteFileInputPlugin.kt
 - src/test/resources/input/host1/test.csv
 - src/test/resources/input/host1/test_command.csv
 - src/test/resources/input/host2/test.csv
 - src/test/resources/input/host2/test_command.csv
+- src/test/resources/script/hosts.sh
 - src/test/resources/yaml/base.yml
 - classpath/bcpkix-jdk15on-1.51.jar
 - classpath/bcprov-jdk15on-1.51.jar
 - classpath/eddsa-0.1.0.jar
-- classpath/embulk-input-remote-0.2.0.jar
+- classpath/embulk-input-remote-0.3.0.jar
 - classpath/jzlib-1.1.3.jar
+- classpath/kotlin-runtime-1.0.6.jar
+- classpath/kotlin-stdlib-1.0.6.jar
 - classpath/sshj-0.19.1.jar
 homepage: https://github.com/kamatama41/embulk-input-remote
 licenses:

data/example/example.yml.liquid

@@ -1,29 +0,0 @@
-in:
-  type: remote
-  hosts:
-    - localhost
-  path: {{ env.PROJECT_ROOT }}/example/csv
-  ignore_not_found_hosts: true
-  auth:
-    user: {{ env.USER }}
-    type: password
-    password: {{ env.PASSWORD }}
-  decoders:
-  - {type: gzip}
-  parser:
-    charset: UTF-8
-    newline: CRLF
-    type: csv
-    delimiter: ','
-    quote: '"'
-    trim_if_not_quoted: false
-    skip_header_lines: 1
-    allow_extra_columns: false
-    allow_optional_columns: false
-    columns:
-    - {name: id, type: long}
-    - {name: account, type: long}
-    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
-    - {name: purchase, type: timestamp, format: '%Y%m%d'}
-    - {name: comment, type: string}
-out: {type: stdout}

data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java

@@ -1,314 +0,0 @@
-package org.embulk.input;
-
-import java.io.BufferedReader;
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Objects;
-
-import com.fasterxml.jackson.annotation.JsonCreator;
-import com.fasterxml.jackson.annotation.JsonProperty;
-import com.google.common.base.Optional;
-import com.google.common.collect.ImmutableList;
-import org.embulk.config.Config;
-import org.embulk.config.ConfigDefault;
-import org.embulk.config.ConfigDiff;
-import org.embulk.config.ConfigInject;
-import org.embulk.config.ConfigSource;
-import org.embulk.config.Task;
-import org.embulk.config.TaskReport;
-import org.embulk.config.TaskSource;
-import org.embulk.input.remote.SSHClient;
-import org.embulk.spi.BufferAllocator;
-import org.embulk.spi.Exec;
-import org.embulk.spi.FileInputPlugin;
-import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamTransactionalFileInput;
-import org.slf4j.Logger;
-
-public class RemoteFileInputPlugin
-		implements FileInputPlugin {
-	public interface PluginTask
-			extends Task {
-		@Config("hosts")
-		@ConfigDefault("[]")
-		List<String> getHosts();
-
-		@Config("hosts_command")
-		@ConfigDefault("null")
-		Optional<String> getHostsCommand();
-
-		@Config("hosts_separator")
-		@ConfigDefault("\" \"")
-		String getHostsSeparator();
-
-		@Config("default_port")
-		@ConfigDefault("22")
-		int getDefaultPort();
-
-		@Config("path")
-		@ConfigDefault("\"\"")
-		String getPath();
-
-		@Config("path_command")
-		@ConfigDefault("null")
-		Optional<String> getPathCommand();
-
-		@Config("auth")
-		AuthConfig getAuthConfig();
-
-		@Config("ignore_not_found_hosts")
-		@ConfigDefault("false")
-		boolean getIgnoreNotFoundHosts();
-
-		@Config("done_targets")
-		@ConfigDefault("[]")
-		List<Target> getDoneTargets();
-
-		void setDoneTargets(List<Target> lastTarget);
-
-		List<Target> getTargets();
-
-		void setTargets(List<Target> targets);
-
-		@ConfigInject
-		BufferAllocator getBufferAllocator();
-	}
-
-	public interface AuthConfig extends Task {
-		@Config("type")
-		@ConfigDefault("\"public_key\"")
-		String getType();
-
-		@Config("user")
-		@ConfigDefault("null")
-		Optional<String> getUser();
-
-		@Config("key_path")
-		@ConfigDefault("null")
-		Optional<String> getKeyPath();
-
-		@Config("password")
-		@ConfigDefault("null")
-		Optional<String> getPassword();
-
-		@Config("skip_host_key_verification")
-		@ConfigDefault("false")
-		boolean getSkipHostKeyVerification();
-	}
-
-	private final Logger log = Exec.getLogger(getClass());
-
-	@Override
-	public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control) {
-		PluginTask task = config.loadConfig(PluginTask.class);
-		List<Target> targets = listTargets(task);
-		log.info("Loading targets {}", targets);
-		task.setTargets(targets);
-
-		// number of processors is same with number of targets
-		int taskCount = targets.size();
-		return resume(task.dump(), taskCount, control);
-	}
-
-	private List<Target> listTargets(PluginTask task) {
-		final List<String> hosts = listHosts(task);
-		final String path = getPath(task);
-
-		final ImmutableList.Builder<Target> builder = ImmutableList.builder();
-		List<Target> doneTargets = task.getDoneTargets();
-		for (String host : hosts) {
-			String[] split = host.split(":");
-			final String targetHost = split[0];
-			int targetPort = task.getDefaultPort();
-			if (split.length > 1) {
-				targetPort = Integer.valueOf(split[1]);
-			}
-			Target target = new Target(targetHost, targetPort, path);
-
-			if (!doneTargets.contains(target)) {
-				if (task.getIgnoreNotFoundHosts()) {
-					try {
-						final boolean exists = exists(target, task);
-						if (!exists) {
-							continue;
-						}
-					} catch (IOException e) {
-						log.warn("failed to check the file exists. " + target.toString(), e);
-						continue;
-					}
-				}
-				builder.add(target);
-			}
-		}
-		return builder.build();
-	}
-
-	private List<String> listHosts(PluginTask task) {
-		final String hostsCommand = task.getHostsCommand().orNull();
-		if (hostsCommand != null) {
-			final String stdout = execCommand(hostsCommand).trim();
-			return Arrays.asList(stdout.split(task.getHostsSeparator()));
-		} else {
-			return task.getHosts();
-		}
-	}
-
-	private String getPath(PluginTask task) {
-		final String pathCommand = task.getPathCommand().orNull();
-		if (pathCommand != null) {
-			return execCommand(pathCommand).trim();
-		} else {
-			return task.getPath();
-		}
-	}
-
-	private String execCommand(String command) {
-		ProcessBuilder pb = new ProcessBuilder("sh", "-c", command);    // TODO: windows
-		log.info("Running command {}", command);
-		try {
-			final Process process = pb.start();
-			try (InputStream stream = process.getInputStream();
-				 BufferedReader brStdout = new BufferedReader(new InputStreamReader(stream))
-			) {
-				String line;
-				StringBuilder stdout = new StringBuilder();
-				while ((line = brStdout.readLine()) != null) {
-					stdout.append(line);
-				}
-
-				final int code = process.waitFor();
-				if (code != 0) {
-					throw new IOException(String.format(
-							"Command finished with non-zero exit code. Exit code is %d.", code));
-				}
-
-				return stdout.toString();
-			}
-		} catch (IOException | InterruptedException e) {
-			throw new RuntimeException(e);
-		}
-	}
-
-	@Override
-	public ConfigDiff resume(TaskSource taskSource,
-							 int taskCount,
-							 FileInputPlugin.Control control) {
-		PluginTask task = taskSource.loadTask(PluginTask.class);
-
-		control.run(taskSource, taskCount);
-
-		List<Target> targets = new ArrayList<>(task.getTargets());
-
-		return Exec.newConfigDiff().set("done_targets", targets);
-	}
-
-	@Override
-	public void cleanup(TaskSource taskSource,
-						int taskCount,
-						List<TaskReport> successTaskReports) {
-	}
-
-	@Override
-	public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
-		final PluginTask task = taskSource.loadTask(PluginTask.class);
-		final Target target = task.getTargets().get(taskIndex);
-
-		return new InputStreamTransactionalFileInput(
-				task.getBufferAllocator(),
-				new InputStreamTransactionalFileInput.Opener() {
-					@Override
-					public InputStream open() throws IOException {
-						return download(target, task);
-					}
-				}
-		) {
-			@Override
-			public void abort() {
-
-			}
-
-			@Override
-			public TaskReport commit() {
-				return Exec.newTaskReport();
-			}
-		};
-	}
-
-	private boolean exists(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
-			final String checkCmd = "ls " + target.getPath();    // TODO: windows
-			final int timeout = 5/* second */;
-			final SSHClient.CommandResult commandResult = client.execCommand(checkCmd, timeout);
-
-			if(commandResult.getStatus() != 0) {
-				log.warn("Remote file not found. {}", target.toString());
-				return false;
-			} else {
-				return true;
-			}
-		}
-	}
-
-	private InputStream download(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = SSHClient.connect(target.getHost(), target.getPort(), task.getAuthConfig())) {
-			final ByteArrayOutputStream stream = new ByteArrayOutputStream();
-			client.scpDownload(target.getPath(), stream);
-			return new ByteArrayInputStream(stream.toByteArray());
-		}
-	}
-
-	public static class Target {
-		private final String host;
-		private final int port;
-		private final String path;
-
-		@JsonCreator
-		public Target(
-				@JsonProperty("host") String host,
-				@JsonProperty("port") int port,
-				@JsonProperty("path") String path
-		) {
-			this.host = host;
-			this.port = port;
-			this.path = path;
-		}
-
-		public String getHost() {
-			return host;
-		}
-
-		public int getPort() {
-			return port;
-		}
-
-		public String getPath() {
-			return path;
-		}
-
-		@Override
-		public boolean equals(Object o) {
-			if (this == o) return true;
-			if (o == null || getClass() != o.getClass()) return false;
-			Target target = (Target) o;
-			return port == target.port &&
-					Objects.equals(host, target.host) &&
-					Objects.equals(path, target.path);
-		}
-
-		@Override
-		public int hashCode() {
-			return Objects.hash(host, port, path);
-		}
-
-		@Override
-		public String toString() {
-			return host + ":" + port + ":" + path;
-		}
-	}
-}

data/src/main/java/org/embulk/input/remote/SSHClient.java

@@ -1,116 +0,0 @@
-package org.embulk.input.remote;
-
-import net.schmizz.sshj.DefaultConfig;
-import net.schmizz.sshj.connection.channel.direct.Session;
-import net.schmizz.sshj.transport.verification.PromiscuousVerifier;
-import net.schmizz.sshj.xfer.InMemoryDestFile;
-import net.schmizz.sshj.xfer.LocalDestFile;
-import org.embulk.input.RemoteFileInputPlugin;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.concurrent.TimeUnit;
-
-public class SSHClient implements Closeable {
-
-	private final net.schmizz.sshj.SSHClient client;
-
-	public static SSHClient connect(
-			String host, int port, RemoteFileInputPlugin.AuthConfig authConfig
-	) throws IOException {
-
-		SSHClient client =  new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig()));
-		client.connectToHost(host, port, authConfig);
-		return client;
-	}
-
-	private SSHClient(net.schmizz.sshj.SSHClient client) {
-		this.client = client;
-	}
-
-	private void connectToHost(String host, int port, RemoteFileInputPlugin.AuthConfig authConfig) throws IOException {
-		if (authConfig.getSkipHostKeyVerification()) {
-			client.addHostKeyVerifier(new PromiscuousVerifier());
-		}
-		client.loadKnownHosts();
-		client.connect(host, port);
-
-		final String type = authConfig.getType();
-		final String user = authConfig.getUser().or(System.getProperty("user.name"));
-
-		if ("password".equals(type)) {
-			if (authConfig.getPassword().isPresent()) {
-				client.authPassword(user, authConfig.getPassword().get());
-			} else {
-				throw new IllegalStateException("Password is not set.");
-			}
-		} else if ("public_key".equals(type)) {
-			if (authConfig.getKeyPath().isPresent()) {
-				client.authPublickey(user, authConfig.getKeyPath().get());
-			} else {
-				client.authPublickey(user);
-			}
-		} else {
-			throw new UnsupportedOperationException("Unsupported auth type : " + type);
-		}
-	}
-
-	public CommandResult execCommand(String command, int timeoutSecond) throws IOException {
-		try (final Session session = client.startSession()) {
-			final Session.Command cmd = session.exec(command);
-			cmd.join(timeoutSecond, TimeUnit.SECONDS);
-			return new CommandResult(cmd.getExitStatus(), cmd.getInputStream());
-		}
-	}
-
-	public void scpDownload(String path, OutputStream stream) throws IOException {
-		client.useCompression();
-		client.newSCPFileTransfer().download(path, new InMemoryDestFileImpl(stream));
-	}
-
-	private static class InMemoryDestFileImpl extends InMemoryDestFile {
-
-		private OutputStream outputStream;
-
-		public InMemoryDestFileImpl(OutputStream outputStream) {
-			this.outputStream = outputStream;
-		}
-
-		@Override
-		public OutputStream getOutputStream() throws IOException {
-			return outputStream;
-		}
-
-		@Override
-		public LocalDestFile getTargetDirectory(String dirname) throws IOException {
-			return this;
-		}
-	}
-
-	@Override
-	public void close() throws IOException {
-		if (client != null) {
-			client.close();
-		}
-	}
-
-	public static class CommandResult {
-		int status;
-		InputStream stdout;
-
-		private CommandResult(int status, InputStream stdout) {
-			this.status = status;
-			this.stdout = stdout;
-		}
-
-		public int getStatus() {
-			return status;
-		}
-
-		public InputStream getStdout() {
-			return stdout;
-		}
-	}
-}