embulk-input-sftp 0.1.0

data/src/main/java/org/embulk/input/sftp/SftpFileInputPlugin.java

@@ -0,0 +1,67 @@
+package org.embulk.input.sftp;
+
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.TransactionalFileInput;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+public class SftpFileInputPlugin
+        implements FileInputPlugin
+{
+    @Override
+    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    {
+        PluginTask task = config.loadConfig(PluginTask.class);
+
+        // list files recursively
+        task.setFiles(SftpFileInput.listFilesByPrefix(task));
+        // number of processors is same with number of files
+        return resume(task.dump(), task.getFiles().size(), control);
+    }
+
+    @Override
+    public ConfigDiff resume(TaskSource taskSource,
+                             int taskCount,
+                             FileInputPlugin.Control control)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+
+        control.run(taskSource, taskCount);
+
+        ConfigDiff configDiff = Exec.newConfigDiff();
+
+        List<String> files = new ArrayList<String>(task.getFiles());
+        if (files.isEmpty()) {
+            if (task.getLastPath().isPresent()) {
+                configDiff.set("last_path", task.getLastPath().get());
+            }
+        }
+        else {
+            Collections.sort(files);
+            configDiff.set("last_path", files.get(files.size() - 1));
+        }
+
+        return configDiff;
+    }
+
+    @Override
+    public void cleanup(TaskSource taskSource,
+                        int taskCount,
+                        List<TaskReport> successTaskReports)
+    {
+    }
+
+    @Override
+    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    {
+        PluginTask task = taskSource.loadTask(PluginTask.class);
+        return new SftpFileInput(task, taskIndex);
+    }
+}

data/src/main/java/org/embulk/input/sftp/SingleFileProvider.java

@@ -0,0 +1,71 @@
+package org.embulk.input.sftp;
+
+import org.apache.commons.vfs2.FileObject;
+import org.apache.commons.vfs2.FileSystemException;
+import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.embulk.spi.Exec;
+import org.embulk.spi.util.InputStreamFileInput;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public class SingleFileProvider
+        implements InputStreamFileInput.Provider
+{
+    private final StandardFileSystemManager manager;
+    private final String key;
+    private final PluginTask task;
+    private final int maxConnectionRetry;
+    private boolean opened = false;
+    private final Logger log = Exec.getLogger(SingleFileProvider.class);
+
+    public SingleFileProvider(PluginTask task, int taskIndex, StandardFileSystemManager manager)
+    {
+        this.manager = manager;
+        this.key = task.getFiles().get(taskIndex);
+        this.task = task;
+        this.maxConnectionRetry = task.getMaxConnectionRetry();
+    }
+
+    @Override
+    public InputStream openNext() throws IOException
+    {
+        if (opened) {
+            return null;
+        }
+        opened = true;
+
+        int count = 0;
+        while (true) {
+            try {
+                FileObject file = manager.resolveFile(key, SftpFileInput.initializeFsOptions(task));
+                log.info("Starting to download file {}", key);
+
+                return file.getContent().getInputStream();
+            }
+            catch (FileSystemException ex) {
+                if (++count == maxConnectionRetry) {
+                    throw ex;
+                }
+                log.warn("failed to connect sftp server: " + ex.getMessage(), ex);
+
+                try {
+                    long sleepTime = ((long) Math.pow(2, count) * 1000);
+                    log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
+                    Thread.sleep(sleepTime); // milliseconds
+                }
+                catch (InterruptedException ex2) {
+                    // Ignore this exception because this exception is just about `sleep`.
+                    log.warn(ex2.getMessage(), ex2);
+                }
+                log.warn("retrying to connect sftp server: " + count + " times");
+            }
+        }
+    }
+
+    @Override
+    public void close()
+    {
+    }
+}

data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java

@@ -0,0 +1,5 @@
+package org.embulk.input.sftp;
+
+public class TestSftpFileInputPlugin
+{
+}

data/src/test/resources/id_rsa

@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-128-CBC,D2F2C35989F5D5F3CA9482CFDE382598
+
+k11GNDLfBbb3QV1GZs/78aMiznfXE3ZuJhSm3W9pWRq5h2DAxOTKYUuTvlbUtlQt
+N5y708MCynftaeARdCTjnoDRhm2SHHc7JiN4g1cBuVaqst3SOxZvbITrtW0xqW0q
+XvEiud5RSUwwXgjUPJFXNtchQOhYvsm7HzUOH8K8E6if65dD7gyekF8Smo1alRzB
+p40FVty81elOO5+tFJuXJTxayNcUh8JHjrFTUToN8zAiAkIkvqd9JZe+jTRZjoYr
+H/yFiWm6sHFEHsAPnvdYjDl870pGs1qqm74CZKu383C5ioFd9aK7LaZBi5gvz8HX
+anLK1DZSEUdhqSBAHEP0IvLvFNtY/5Hc0ohQt+BHOpnLjoSzckUJjly8bzmYrT1L
+uXBLYkLnFaFz4t/HowSfF8WgRqUh8dDcXDWuXkvMy5WV6YII5hx0/PN8F9mE78Jc
+jeIcuQBV7MuBaJDmpktB2DH+Zt1T/391MGTvU3ZwaNcoDCuavt9uR1OtgTXvbw7w
+qY/uEfasVzXxs6paRY6E85uUuFQy2XQIBqfermQYBg/vncAIWRfXh5AcgKktE0XH
+LczzalWwdzakCJ816htYDe4311K0oPjxStEMFGW390MuKMnpyK2IDBTBub5tzcMl
+BGCtNoJUrPj6Ty/vIhiADEMgs63C5sejyVnL9CiJGC/we0Ihe8akY5l2h9VV+C+8
+80qQjdNU2yGu8Jta1avFmzu/1wWN3KAO/GujjbhC8thAizu1Of62q+FH+sbni9Lr
+raJypD1iVsvIkNlwQpAJVJz4mFgLvJBIazc5unWrmOLdUolQ9/7rCSKXUBHF1I7G
+5MW1YOyBmgNbagCNoE1W4VBF9vafBxtoi1FYPP1uM0JbXAEZLGSWcm9N8hlxx+EP
+AxATHpOBxgUYHUJK9hlga0AlZ9+BSU/WhDdLtYgFz9cm7b1571wPPAReXYMuip+E
+Rh0FGWTQfRZ20Vqh5wt4zat7wvCz65dh/SPiZ+H1/qMTpX3LF9dH/4++H13k2xPb
+P78L74i4tPLpKnU0RmjZc652SCQ2th8DEihkPrHDrClWDuZoSx/b2gQEq26+ys7P
+1Ftvjdejh13ZyY4IsVzfRtABU5ma3iHUoL09wYZFi2TAfSEq+zNpNg3MwCILhCs/
+TEM2zXPUJgVoxVgWp1Vfu+1MMubXSShsz3R6Ec/AC22PdpIu6h/7twPZOD+zjOuv
+r6lbQs33OLmhp1I+7xy9CNRHCjm7GExk/l3ZudCxJGyQ/a4s8iSiIhJ0R8LbX3Ux
+VT/ZAQjjxAMvCEpGcuricrVbo+QXF5epgtV73bQj99LCYtS4rFk6Pwn/06IxbKlN
+/V+ltFNSexyxRFcBxiq5e7G2zQEcdNLAFXvAXHk6nULUWuY5Nz0ugdUyJVytrNGk
+Sd2501Lz6qzH5WpLcg3RB+i5aTE+SuAZHOpaBqL9YHMZ2TFtCfV/iTJQEFG/kYdL
+Skiu+dcYjO4kSiLsznxSAPGvmZrUHFk5f2MzEuCw+ibSfsO5IEcjg9PD+zXbbzTc
+VK8MzAv8y+3OG6M4pzMAwEsxOUgflyVZgjEA1lOPplkmj3Q0NOgipczi28qNUYeh
+-----END RSA PRIVATE KEY-----

data/src/test/resources/id_rsa.pub

@@ -0,0 +1 @@
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCtW2TpgsOXiZXmIycRsrRKN/IC8LeLSIkDBs23ut4dvLn0sNTdY7C5QdemFHN2zqzHI85U0kKZ5VorEXF/bfbRImXSEWafhGNN/DsdLNGpIqarXVsQ9j8/Yt+KngubjBM/uNZt62zBNk+jyU7OyVAyHtjW7+9uBl7ApGGiIEBOJZJcTFkbUiXqwzWtEtsOPaEdrc7GVJaJ8bwARptYbn32Z8gFNHwQyBBQiTXWsGX42EfYYs1HcJlH3GkDLVOsvrgHAN80dDzrJrvSTsFURThnvrYezubV4y4EvRhwF9BUL8b+Xz8senNpWEla/Qv1IQLSB/UwUhIiXYtpamS7NIFx

metadata

@@ -0,0 +1,95 @@
+--- !ruby/object:Gem::Specification
+name: embulk-input-sftp
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Satoshi Akama
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: bundler
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  name: rake
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
+description: Reads files stored on remote server using SFTP.
+email:
+- satoshiakama@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- README.md
+- build.gradle
+- config/checkstyle/checkstyle.xml
+- config/checkstyle/default.xml
+- gradle/wrapper/gradle-wrapper.jar
+- gradle/wrapper/gradle-wrapper.properties
+- gradlew
+- gradlew.bat
+- lib/embulk/input/sftp.rb
+- src/main/java/org/embulk/input/sftp/PluginTask.java
+- src/main/java/org/embulk/input/sftp/ProxyTask.java
+- src/main/java/org/embulk/input/sftp/SftpFileInput.java
+- src/main/java/org/embulk/input/sftp/SftpFileInputPlugin.java
+- src/main/java/org/embulk/input/sftp/SingleFileProvider.java
+- src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java
+- src/test/resources/id_rsa
+- src/test/resources/id_rsa.pub
+- classpath/commons-io-1.3.2.jar
+- classpath/commons-logging-1.2.jar
+- classpath/commons-vfs2-2.1.1660580.2.jar
+- classpath/embulk-input-sftp-0.1.0.jar
+- classpath/jsch-0.1.53.jar
+homepage: https://github.com/sakama/embulk-input-sftp
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: SFTP file input plugin for Embulk
+test_files: []