embulk-input-remote 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e30d0f99dcb958e08afc14b19630f2eab518c01f
-  data.tar.gz: 086290764bf8c788db1b8b2e705f9a1eebacb49e
+  metadata.gz: 303e89aa29cb5154cae5337a90dae5c3dad30e70
+  data.tar.gz: 0447d693207db43d2f3058139a43d5e9a225e117
 SHA512:
-  metadata.gz: 186acc6cf0ff2ef037819d0277f24b03760038515bf377334afcfce89918ab1bb8424a0f726047ad9e79bbef138e71c66b5fac78d17e1df420407cc0213e207c
-  data.tar.gz: d1090d03a03e2a4708897989dbda36d60d86fd4226a6b7893cc9bb9f72ef11080b7c69f79280c57374a0bc59f3496e64cb69ddcce9f3085d460b59e15d593f8e
+  metadata.gz: 0caa5463e3c5f7573ddc72b5640ab01702e307cf3cd26001f16ea71aa382be58de3c8d6b06c138d4d829d50843125af7cc9996b5a73b5ac08bffa05de5200ecd
+  data.tar.gz: 146c22cf5416e4cd47d059f121b0a054eddad875eeddd30b4af178d7c09e888762aae2875930d01a0f43450afc96e88a2b913895b65f85ebc0bb1dc1de20a1a6

data/build.gradle

@@ -13,13 +13,13 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.1"
-    compile  "com.hierynomus:sshj:0.11.0"
+    compile  "org.embulk:embulk-core:0.7.10"
+    compile  "com.hierynomus:sshj:0.15.0"
     compile  "com.jcraft:jzlib:1.1.3"
-    provided "org.embulk:embulk-core:0.6.1"
+    provided "org.embulk:embulk-core:0.7.10"
     testCompile "junit:junit:4.+"
 }
 
@@ -40,7 +40,7 @@ task gemspec << { file("build/gemspec").write($/
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["Shinichi ISHIMURA"]
+  spec.authors       = ["Shinichi Ishimura"]
   spec.summary       = %[Remote file input plugin for Embulk]
   spec.description   = %[Reads files stored on Remote hosts by SCP.]
   spec.email         = ["shiketaudonko41@gmail.com"]

data/example/example.yml.liquid

@@ -0,0 +1,29 @@
+in:
+  type: remote
+  hosts:
+    - localhost
+  path: {{ env.PROJECT_ROOT }}/example/csv
+  ignore_not_found_hosts: true
+  auth:
+    user: {{ env.USER }}
+    type: password
+    password: {{ env.PASSWORD }}
+  decoders:
+  - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    trim_if_not_quoted: false
+    skip_header_lines: 1
+    allow_extra_columns: false
+    allow_optional_columns: false
+    columns:
+    - {name: id, type: long}
+    - {name: account, type: long}
+    - {name: time, type: timestamp, format: '%Y-%m-%d %H:%M:%S'}
+    - {name: purchase, type: timestamp, format: '%Y%m%d'}
+    - {name: comment, type: string}
+out: {type: stdout}

data/gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
-#Mon Apr 13 17:43:10 JST 2015
+#Sun Dec 13 00:26:07 JST 2015
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.2.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.6-bin.zip

data/gradlew.bat

@@ -1,90 +1,90 @@
-@if "%DEBUG%" == "" @echo off
-@rem ##########################################################################
-@rem
-@rem  Gradle startup script for Windows
-@rem
-@rem ##########################################################################
-
-@rem Set local scope for the variables with windows NT shell
-if "%OS%"=="Windows_NT" setlocal
-
-@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
-set DEFAULT_JVM_OPTS=
-
-set DIRNAME=%~dp0
-if "%DIRNAME%" == "" set DIRNAME=.
-set APP_BASE_NAME=%~n0
-set APP_HOME=%DIRNAME%
-
-@rem Find java.exe
-if defined JAVA_HOME goto findJavaFromJavaHome
-
-set JAVA_EXE=java.exe
-%JAVA_EXE% -version >NUL 2>&1
-if "%ERRORLEVEL%" == "0" goto init
-
-echo.
-echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:findJavaFromJavaHome
-set JAVA_HOME=%JAVA_HOME:"=%
-set JAVA_EXE=%JAVA_HOME%/bin/java.exe
-
-if exist "%JAVA_EXE%" goto init
-
-echo.
-echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
-echo.
-echo Please set the JAVA_HOME variable in your environment to match the
-echo location of your Java installation.
-
-goto fail
-
-:init
-@rem Get command-line arguments, handling Windowz variants
-
-if not "%OS%" == "Windows_NT" goto win9xME_args
-if "%@eval[2+2]" == "4" goto 4NT_args
-
-:win9xME_args
-@rem Slurp the command line arguments.
-set CMD_LINE_ARGS=
-set _SKIP=2
-
-:win9xME_args_slurp
-if "x%~1" == "x" goto execute
-
-set CMD_LINE_ARGS=%*
-goto execute
-
-:4NT_args
-@rem Get arguments from the 4NT Shell from JP Software
-set CMD_LINE_ARGS=%$
-
-:execute
-@rem Setup the command line
-
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
-
-@rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
-
-:end
-@rem End local scope for the variables with windows NT shell
-if "%ERRORLEVEL%"=="0" goto mainEnd
-
-:fail
-rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
-rem the _cmd.exe /c_ return code!
-if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
-exit /b 1
-
-:mainEnd
-if "%OS%"=="Windows_NT" endlocal
-
-:omega
+@if "%DEBUG%" == "" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS=
+
+set DIRNAME=%~dp0
+if "%DIRNAME%" == "" set DIRNAME=.
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if "%ERRORLEVEL%" == "0" goto init
+
+echo.
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto init
+
+echo.
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
+echo.
+echo Please set the JAVA_HOME variable in your environment to match the
+echo location of your Java installation.
+
+goto fail
+
+:init
+@rem Get command-line arguments, handling Windowz variants
+
+if not "%OS%" == "Windows_NT" goto win9xME_args
+if "%@eval[2+2]" == "4" goto 4NT_args
+
+:win9xME_args
+@rem Slurp the command line arguments.
+set CMD_LINE_ARGS=
+set _SKIP=2
+
+:win9xME_args_slurp
+if "x%~1" == "x" goto execute
+
+set CMD_LINE_ARGS=%*
+goto execute
+
+:4NT_args
+@rem Get arguments from the 4NT Shell from JP Software
+set CMD_LINE_ARGS=%$
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
+
+:end
+@rem End local scope for the variables with windows NT shell
+if "%ERRORLEVEL%"=="0" goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+if  not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
+exit /b 1
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

data/src/main/java/org/embulk/input/RemoteFileInputPlugin.java

@@ -16,20 +16,20 @@ import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import com.google.common.base.Optional;
 import com.google.common.collect.ImmutableList;
-import org.embulk.config.CommitReport;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
 import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.input.remote.SSHClient;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
-import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamTransactionalFileInput;
 import org.slf4j.Logger;
 
 import javax.annotation.Nullable;
@@ -183,64 +183,37 @@ public class RemoteFileInputPlugin
 	@Override
 	public void cleanup(TaskSource taskSource,
 						int taskCount,
-						List<CommitReport> successCommitReports) {
+						List<TaskReport> successTaskReports) {
 	}
 
 	@Override
 	public TransactionalFileInput open(TaskSource taskSource, int taskIndex) {
-		PluginTask task = taskSource.loadTask(PluginTask.class);
+		final PluginTask task = taskSource.loadTask(PluginTask.class);
 		final Target target = task.getTargets().get(taskIndex);
 
-		try {
-			return new PluginFileInput(task, download(target, task));
-		} catch (IOException e) {
-			throw new RuntimeException(e);
-		}
-	}
-
-	public static class PluginFileInput
-			extends InputStreamFileInput
-			implements TransactionalFileInput {
-		// TODO create single-file InputStreamFileInput utility
-		private static class RemoteFileProvider
-				implements InputStreamFileInput.Provider {
-			private boolean opened = false;
-			private final InputStream input;
-
-			public RemoteFileProvider(InputStream input) {
-				this.input = input;
-			}
-
-			@Override
-			public InputStream openNext() throws IOException {
-				if (opened) {
-					return null;
+		return new InputStreamTransactionalFileInput(
+				task.getBufferAllocator(),
+				new InputStreamTransactionalFileInput.Opener() {
+					@Override
+					public InputStream open() throws IOException {
+						return download(target, task);
+					}
 				}
-				opened = true;
-				return input;
+		) {
+			@Override
+			public void abort() {
+
 			}
 
 			@Override
-			public void close() {
+			public TaskReport commit() {
+				return Exec.newTaskReport();
 			}
-		}
-
-		public PluginFileInput(PluginTask task, InputStream input) {
-			super(task.getBufferAllocator(), new RemoteFileProvider(input));
-		}
-
-		@Override
-		public void abort() {
-		}
-
-		@Override
-		public CommitReport commit() {
-			return Exec.newCommitReport();
-		}
+		};
 	}
 
 	private boolean exists(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = new SSHClient()) {
+		try (SSHClient client = SSHClient.getInstance()) {
 			client.connect(target.getHost(), task.getAuth());
 
 			final String checkCmd = "ls " + target.getPath();    // TODO: windows
@@ -257,7 +230,7 @@ public class RemoteFileInputPlugin
 	}
 
 	private InputStream download(Target target, PluginTask task) throws IOException {
-		try (SSHClient client = new SSHClient()) {
+		try (SSHClient client = SSHClient.getInstance()) {
 			client.connect(target.getHost(), task.getAuth());
 			final ByteArrayOutputStream stream = new ByteArrayOutputStream();
 			client.scpDownload(target.getPath(), stream);

data/src/main/java/org/embulk/input/remote/SSHClient.java

@@ -1,6 +1,11 @@
 package org.embulk.input.remote;
 
+import com.hierynomus.sshj.signature.SignatureEdDSA;
+import net.schmizz.sshj.DefaultConfig;
 import net.schmizz.sshj.connection.channel.direct.Session;
+import net.schmizz.sshj.signature.SignatureDSA;
+import net.schmizz.sshj.signature.SignatureECDSA;
+import net.schmizz.sshj.signature.SignatureRSA;
 import net.schmizz.sshj.xfer.InMemoryDestFile;
 import net.schmizz.sshj.xfer.LocalDestFile;
 
@@ -15,8 +20,18 @@ public class SSHClient implements Closeable {
 
 	private final net.schmizz.sshj.SSHClient client;
 
-	public SSHClient() {
-		this(new net.schmizz.sshj.SSHClient());
+	public static SSHClient getInstance() {
+		return new SSHClient(new net.schmizz.sshj.SSHClient(new DefaultConfig(){
+			@Override
+			protected void initSignatureFactories() {
+				setSignatureFactories(
+						new SignatureRSA.Factory(),
+						new SignatureECDSA.Factory(),
+						new SignatureDSA.Factory(),
+						new SignatureEdDSA.Factory()
+				);
+			}
+		}));
 	}
 
 	/* package for test */
@@ -26,7 +41,6 @@ public class SSHClient implements Closeable {
 
 	public void connect(String host, Map<String, String> authConfig) throws IOException {
 		client.loadKnownHosts();
-
 		client.connect(host);
 
 		final String type = authConfig.get("type") != null ? authConfig.get("type") : "public_key";

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-remote
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
-- Shinichi ISHIMURA
+- Shinichi Ishimura
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-07-28 00:00:00.000000000 Z
+date: 2015-12-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -49,6 +49,8 @@ files:
 - LICENSE.txt
 - README.md
 - build.gradle
+- example/csv/sample_01.csv.gz
+- example/example.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -59,9 +61,10 @@ files:
 - src/test/java/org/embulk/input/TestRemoteFileInputPlugin.java
 - classpath/bcpkix-jdk15on-1.51.jar
 - classpath/bcprov-jdk15on-1.51.jar
-- classpath/embulk-input-remote-0.1.4.jar
+- classpath/ecc-25519-java-1.0.1.jar
+- classpath/embulk-input-remote-0.1.5.jar
 - classpath/jzlib-1.1.3.jar
-- classpath/sshj-0.11.0.jar
+- classpath/sshj-0.15.0.jar
 homepage: https://github.com/kamatama41/embulk-input-remote
 licenses:
 - MIT