embulk-input-sftp 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 20b62b0b1e6694bdaa818d0a993c69a62762dfe0
-  data.tar.gz: d2cc4ef2c4eb0605cbd5df857840f13559d98112
+  metadata.gz: df31c9ece3349f98ed7d6cb077ea54d93eb03983
+  data.tar.gz: 86a2988731cc55a81ed7e191eb453b952a3d13a4
 SHA512:
-  metadata.gz: 50e595f0263b65e5e360128730ca743ef8d66c2ee0002219e24bd4ff18b55c99516de51399fa4e4b2b9a86998cc0403623d52182bedb8226e95d59bbab82a404
-  data.tar.gz: 67182a732de5b5d001630e99724b9b7b7c24383c17dab5628c67c61e41f3b8fe3bf4904227a881d65aedd962e9a3422f581fdecc8a1b8bd23a9520b54cbe7e84
+  metadata.gz: 5200a610d8dbcde6e46597d9749504da4e1afdeb44f1803c1ff162204ccb96f4eab73397ec889046111b6adf6677653521696fa4e88f4d35e5a0b9c4e3998fe2
+  data.tar.gz: 33a363400d78fd010f238a0270f7e086610ba9a9bc9ea25cf59dcf2d03fb66f09198f5df0aa35ea46545fea2dded4eaa5264cc9a56aaf2c604cb49b424617171

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.4 - 2017-06-05
+
+* [maintenance] Improve logic for remote file search with path_prefix  [#22](https://github.com/sakama/embulk-input-sftp/pull/22)
+
 ## 0.2.3 - 2016-09-30
 
 * [maintenance] Fix auth failure while generating last_path under limited case [#20](https://github.com/sakama/embulk-input-sftp/pull/20)

data/README.md

@@ -135,6 +135,7 @@ in:
 
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
+$ ./gradlew bintrayUpload # release embulk-input-sftp to Bintray maven repo
 ```
 
 ## Test

data/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
+    id "com.jfrog.bintray" version "1.7"
+    id "maven-publish"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
     id "checkstyle"
@@ -14,7 +15,8 @@ configurations {
     provided
 }
 
-version = "0.2.3"
+group = "org.embulk.input.sftp"
+version = "0.2.4"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
@@ -33,6 +35,50 @@ dependencies {
     testCompile "io.netty:netty-all:4.0.34.Final"
 }
 
+javadoc {
+    options {
+        locale = 'en_US'
+        encoding = 'UTF-8'
+    }
+}
+
+// bintray
+bintray {
+    // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
+    user = project.hasProperty('bintray_user') ? bintray_user : ''
+    key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
+
+    publications = ['bintrayMavenRelease']
+    publish = true
+
+    pkg {
+        userOrg = 'embulk-input-sftp'
+        repo = 'maven'
+        name = project.name
+        desc = 'SFTP file input plugin for Embulk'
+        websiteUrl = 'https://github.com/embulk/embulk-input-sftp'
+        issueTrackerUrl = 'https://github.com/embulk/embulk-input-sftp/issues'
+        vcsUrl = 'https://github.com/embulk/embulk-input-sftp.git'
+        licenses = ['Apache-2.0']
+        labels = ['embulk', 'java']
+        publicDownloadNumbers = true
+
+        version {
+            name = project.version
+        }
+    }
+}
+publishing {
+    publications {
+        bintrayMavenRelease(MavenPublication) {
+            from components.java
+            artifact testsJar
+            artifact sourcesJar
+            artifact javadocJar
+        }
+    }
+}
+
 task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
     from (configurations.runtime - configurations.provided + files(jar.archivePath))
@@ -40,6 +86,20 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete "classpath" }
 
+// add tests/javadoc/source jar tasks as artifacts to be released
+task testsJar(type: Jar, dependsOn: classes) {
+    classifier = 'tests'
+    from sourceSets.test.output
+}
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+}
+task javadocJar(type: Jar, dependsOn: javadoc) {
+    classifier = 'javadoc'
+    from javadoc.destinationDir
+}
+
 checkstyle {
     configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
     toolVersion = '6.14.1'

data/src/main/java/org/embulk/input/sftp/SftpFileInput.java

@@ -4,11 +4,15 @@ import com.google.common.base.Function;
 import com.google.common.base.Optional;
 import com.google.common.base.Throwables;
 import org.apache.commons.io.FilenameUtils;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.impl.StandardFileSystemManager;
+import org.apache.commons.vfs2.provider.UriParser;
+import org.apache.commons.vfs2.provider.local.GenericFileNameParser;
 import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
+import org.apache.commons.vfs2.provider.sftp.SftpFileNameParser;
 import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
 import org.embulk.config.ConfigException;
 import org.embulk.config.TaskReport;
@@ -19,7 +23,6 @@ import org.embulk.spi.util.InputStreamFileInput;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
-import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.File;
 import java.io.IOException;
@@ -27,6 +30,8 @@ import java.net.URI;
 import java.net.URISyntaxException;
 import java.util.Arrays;
 
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
+
 public class SftpFileInput
         extends InputStreamFileInput
         implements TransactionalFileInput
@@ -149,76 +154,23 @@ public class SftpFileInput
             if (!uri.isPresent()) {
                 return null;
             }
-            else if (!task.getSecretKeyFile().isPresent() && task.getPassword().isPresent()) {
-                return getRelativePathFromURIwithPassword(task, uri);
-            }
             else {
-                return new URI(uri.get()).getPath();
+                String uriString = uri.get();
+                String scheme = UriParser.extractScheme(uriString);
+                if (StringUtils.isEmpty(scheme)) {
+                    return GenericFileNameParser.getInstance().parseUri(null, null, uriString).getPath();
+                }
+                else if (scheme.equals("sftp")) {
+                    return SftpFileNameParser.getInstance().parseUri(null, null, uriString).getPath();
+                }
+                else {
+                    throw new ConfigException("SFTP Plugin only support SFTP scheme");
+                }
             }
         }
-        catch (URISyntaxException ex) {
-            throw new ConfigException("Failed to generate last_path due to URI parse failure that contains invalid file path.", ex);
-        }
-    }
-
-    private static String getRelativePathFromURIwithPassword(final PluginTask task, final Optional<String> uri)
-    {
-        try {
-            return retryExecutor()
-                    .withRetryLimit(task.getMaxConnectionRetry())
-                    .withInitialRetryWait(500)
-                    .withMaxRetryWait(30 * 1000)
-                    .runInterruptible(new Retryable<String>() {
-                        @Override
-                        public String call() throws URISyntaxException, IOException
-                        {
-                            log.info("Creating last_path from URI contains password in FileList.");
-                            StandardFileSystemManager manager = initializeStandardFileSystemManager();
-
-                            String prefix = new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), null, null, null).toString();
-                            prefix = manager.resolveFile(prefix).toString();
-                            // To avoid URI parse failure when password contains special characters
-                            String newUri = uri.get().replace(prefix, "sftp://user:password@example.com/");
-
-                            return new URI(newUri).getPath();
-                        }
-
-                        @Override
-                        public boolean isRetryableException(Exception exception)
-                        {
-                            if (exception instanceof URISyntaxException) {
-                                // Don't throw cause because URISyntaxException shows password
-                                throw new ConfigException("Failed to generate last_path due to URI parse failure that contains invalid file path or password.");
-                            }
-                            return true;
-                        }
-
-                        @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
-                        {
-                            String message = String.format("SFTP List request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
-                            if (retryCount % 3 == 0) {
-                                log.warn(message, exception);
-                            }
-                            else {
-                                log.warn(message);
-                            }
-                        }
-
-                        @Override
-                        public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
-                        {
-                        }
-                    });
-        }
-        catch (RetryGiveupException ex) {
-            throw new ConfigException("Failed to generate last_path due to SFTP connection failure");
-        }
-        catch (InterruptedException ex) {
-            Throwables.propagate(ex);
+        catch (FileSystemException ex) {
+            throw new ConfigException("Failed to generate last_path due to sftp file name parse failure", ex);
         }
-        return null;
     }
 
     public static FileList listFilesByPrefix(final PluginTask task)
@@ -247,6 +199,7 @@ public class SftpFileInput
                             FileObject files = manager.resolveFile(getSftpFileUri(task, task.getPathPrefix()), fsOptions);
                             String basename = FilenameUtils.getBaseName(task.getPathPrefix());
                             if (files.isFolder()) {
+                                //path_prefix is a folder, we add everything in that folder
                                 FileObject[] children = files.getChildren();
                                 Arrays.sort(children);
                                 for (FileObject f : children) {
@@ -255,7 +208,13 @@ public class SftpFileInput
                                     }
                                 }
                             }
+                            else if (files.isFile()) {
+                                //path_prefix is a file then we just need to add that file
+                                addFileToList(builder, files.toString(), files.getContent().getSize(), "", lastKey);
+                            }
                             else {
+                                // path_prefix is neither file or folder, then we scan the parent folder to file path
+                                // that match the path_prefix basename
                                 FileObject parent = files.getParent();
                                 FileObject[] children = parent.getChildren();
                                 Arrays.sort(children);

data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java

@@ -68,6 +68,9 @@ public class TestSftpFileInputPlugin
     @Rule
     public TemporaryFolder testFolder = new TemporaryFolder();
 
+    @Rule
+    public TemporaryFolder sshDirFolder = new TemporaryFolder();
+
     private Logger log = runtime.getExec().getLogger(TestSftpFileInputPlugin.class);
     private ConfigSource config;
     private SftpFileInputPlugin plugin;
@@ -87,11 +90,12 @@ public class TestSftpFileInputPlugin
     @Before
     public void createResources() throws Exception
     {
+        // set system property for ssh_dir
+        System.setProperty("vfs.sftp.sshdir", sshDirFolder.getRoot().getPath());
         config = config();
         plugin = new SftpFileInputPlugin();
         runner = new FileInputRunner(runtime.getInstance(SftpFileInputPlugin.class));
         output = new MockPageOutput();
-
         if (!log.isDebugEnabled()) {
             // TODO: change logging format: org.apache.commons.logging.Log
             System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
@@ -162,7 +166,7 @@ public class TestSftpFileInputPlugin
     public void testResume()
     {
         PluginTask task = config.loadConfig(PluginTask.class);
-        task.setFiles(createFileList(Arrays.asList("in/aa/a"), task));
+        task.setFiles(createFileList(Arrays.asList("/in/aa/a"), task));
         ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
         {
             @Override
@@ -171,7 +175,7 @@ public class TestSftpFileInputPlugin
                 return emptyTaskReports(taskCount);
             }
         });
-        assertEquals("in/aa/a", configDiff.get(String.class, "last_path"));
+        assertEquals("/in/aa/a", configDiff.get(String.class, "last_path"));
     }
 
     @Test
@@ -373,23 +377,63 @@ public class TestSftpFileInputPlugin
         assertEquals(SftpFileSystemConfigBuilder.PROXY_STREAM, builder.getProxyType(fsOptions));
     }
 
+    /**
+     * Test get relative path with special character password
+     */
     @Test
-    public void testGetRelativePath()
+    public void testGetRelativePathWithPassword()
     {
-        ConfigSource conf = config();
-        String expected = "/path/to/sample.csv";
+        ConfigSource conf = config.deepCopy();
+        String expected = "/path/to/sample !@#.csv";
 
         conf.set("password", "ABCDE");
-        PluginTask task = config.loadConfig(PluginTask.class);
-        String uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample.csv");
+        PluginTask task = conf.loadConfig(PluginTask.class);
+        String uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample !@#.csv");
         assertEquals(expected, SftpFileInput.getRelativePath(task, Optional.of(uri)));
 
         conf.set("password", "ABCD#$¥!%'\"@?<>\\&/_^~|-=+-,{}[]()");
-        task = config.loadConfig(PluginTask.class);
-        uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample.csv");
+        task = conf.loadConfig(PluginTask.class);
+        uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample !@#.csv");
         assertEquals(expected, SftpFileInput.getRelativePath(task, Optional.of(uri)));
     }
 
+    @Test
+    public void testGetRelativePath()
+    {
+        String expected = "/path/to/sample !@#.csv";
+        String path = "/path/to/sample !@#.csv";
+        config.loadConfig(PluginTask.class);
+        assertEquals(expected, SftpFileInput.getRelativePath(null, Optional.of(path)));
+    }
+
+    @Test(expected = ConfigException.class)
+    public void testGetRelativePathWithHttpScheme()
+    {
+        String path = "http://host/path/to/sample !@#.csv";
+        config.loadConfig(PluginTask.class);
+        SftpFileInput.getRelativePath(null, Optional.of(path));
+    }
+
+    /**
+     * When user explicitly set path_prefix to a single file. we should add that file only
+     * @throws Exception
+     */
+    @Test
+    public void testListByPrefixWithSpecificPathPrefix() throws Exception
+    {
+        ConfigSource conf = config.deepCopy();
+        conf.set("path_prefix", REMOTE_DIRECTORY + "sample_01.csv");
+        PluginTask pluginTask = conf.loadConfig(PluginTask.class);
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01 .csv", true);
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01ABC.csv", true);
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01DEF!@#.csv", true);
+        FileList fileList = SftpFileInput.listFilesByPrefix(pluginTask);
+        assertEquals(1, fileList.getTaskCount());
+        assertEquals("sftp://username:password@127.0.0.1:20022/home/username/unittest/sample_01.csv", fileList.get
+                (0).get(0));
+    }
+
     private SshServer createSshServer(String host, int port, final String sshUsername, final String sshPassword)
     {
         // setup a mock sftp server

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sftp
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-30 00:00:00.000000000 Z
+date: 2017-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  prerelease: false
+  type: :development
 description: Reads files stored on remote server using SFTP.
 email:
 - satoshiakama@gmail.com
@@ -72,7 +72,7 @@ files:
 - classpath/commons-io-1.3.2.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.1.jar
-- classpath/embulk-input-sftp-0.2.3.jar
+- classpath/embulk-input-sftp-0.2.4.jar
 - classpath/jsch-0.1.53.jar
 homepage: https://github.com/embulk/embulk-input-sftp
 licenses: