embulk-input-sftp 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2afbf35e073d8e31985a7c797395c796828aa675
-  data.tar.gz: 73667da5ec1be837b3c5811969dc5c4ba4014731
+  metadata.gz: 2437f248e08389b72d6954dbb9da3dd6db93964a
+  data.tar.gz: cb24da548a14d3b986ebb21b1f114f4d69bf2962
 SHA512:
-  metadata.gz: 3bb521683b3668ef39f71233755eaabc3f2b67578e711efe942e095dbf97969f1b7aeeb72bb5cbb422a7eeb3d4dc9384e549e0a9f94a331a3829c0d5d779bb8a
-  data.tar.gz: cbb3f79afc30b70fca469bd6012653d13e58983d03934a48d76e474c277e9cc7b79161a03eaf48b38c2ce7389048545cd0dfa77c7d373a216eec556e84941aee
+  metadata.gz: 240d0bf66262aabc11f90fa0f53c8e83b11253a2cee7cf384d739804d7dfa074ffe3986f703f1d2fed803bc45c69b1d796ec37c736c6d2fd738e153e4710adf6
+  data.tar.gz: 5ae478d9b5d8ce6429c56579b263d84f25214e1f38442e13494e3d6e49216fea3efa9c6eaa7d0754020583eee80d749a8031b2b7912aabab6d8e8c7332014568

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.2.1 - 2016-09-12
+* [maintenance] Fix last_path generation failure when password contains special chars [#15](https://github.com/sakama/embulk-input-sftp/pull/15)
+
 ## 0.2.0 - 2016-08-19
 
 * [new feature] Support incremental option [#11](https://github.com/sakama/embulk-input-sftp/pull/11)

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.2.0"
+version = "0.2.1"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/sftp/SftpFileInput.java

@@ -143,19 +143,82 @@ public class SftpFileInput
         }
     }
 
-    public static String getRelativePath(Optional<String> path)
+    public static String getRelativePath(PluginTask task, Optional<String> uri)
     {
         try {
-            if (path.isPresent()) {
-                return new URI(path.get()).getPath();
+            if (!uri.isPresent()) {
+                return null;
+            }
+            else if (task.getPassword().isPresent()) {
+                return getRelativePathFromURIwithPassword(task, uri);
             }
             else {
-                return null;
+                return new URI(uri.get()).getPath();
             }
         }
         catch (URISyntaxException ex) {
-            return null;
+            throw new ConfigException("Failed to generate last_path due to URI parse failure that contains invalid file path.", ex);
+        }
+    }
+
+    private static String getRelativePathFromURIwithPassword(final PluginTask task, final Optional<String> uri)
+    {
+        try {
+            return retryExecutor()
+                    .withRetryLimit(task.getMaxConnectionRetry())
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<String>() {
+                        @Override
+                        public String call() throws URISyntaxException, IOException
+                        {
+                            log.info("Creating last_path from URI contains password in FileList.");
+                            StandardFileSystemManager manager = initializeStandardFileSystemManager();
+
+                            String prefix = new URI("sftp", initializeUserInfo(task), task.getHost(), task.getPort(), null, null, null).toString();
+                            prefix = manager.resolveFile(prefix).toString();
+                            // To avoid URI parse failure when password contains special characters
+                            String newUri = uri.get().replace(prefix, "sftp://user:password@example.com/");
+
+                            return new URI(newUri).getPath();
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            if (exception instanceof URISyntaxException) {
+                                // Don't throw cause because URISyntaxException shows password
+                                throw new ConfigException("Failed to generate last_path due to URI parse failure that contains invalid file path or password.");
+                            }
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
+                        {
+                            String message = String.format("SFTP List request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            throw new ConfigException("Failed to generate last_path due to FTP connection failure");
+        }
+        catch (InterruptedException ex) {
+            Throwables.propagate(ex);
         }
+        return null;
     }
 
     public static FileList listFilesByPrefix(final PluginTask task)

data/src/main/java/org/embulk/input/sftp/SftpFileInputPlugin.java

@@ -30,11 +30,15 @@ public class SftpFileInputPlugin
                              FileInputPlugin.Control control)
     {
         PluginTask task = taskSource.loadTask(PluginTask.class);
+        String lastPath = null;
+        if (task.getIncremental()) {
+            lastPath = SftpFileInput.getRelativePath(task, task.getFiles().getLastPath(task.getLastPath()));
+        }
         control.run(taskSource, taskCount);
 
         ConfigDiff configDiff = Exec.newConfigDiff();
-        if (task.getIncremental()) {
-            configDiff.set("last_path", SftpFileInput.getRelativePath(task.getFiles().getLastPath(task.getLastPath())));
+        if (task.getIncremental() && lastPath != null) {
+            configDiff.set("last_path", lastPath);
         }
 
         return configDiff;

data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java

@@ -227,7 +227,7 @@ public class TestSftpFileInputPlugin
 
         assertEquals(expected.get(0), actual.get(0));
         assertEquals(expected.get(1), actual.get(1));
-        assertEquals(SftpFileInput.getRelativePath(Optional.of(expected.get(1).get(0))), configDiff.get(String.class, "last_path"));
+        assertEquals(SftpFileInput.getRelativePath(task, Optional.of(expected.get(1).get(0))), configDiff.get(String.class, "last_path"));
     }
 
     @Test
@@ -373,6 +373,23 @@ public class TestSftpFileInputPlugin
         assertEquals(SftpFileSystemConfigBuilder.PROXY_STREAM, builder.getProxyType(fsOptions));
     }
 
+    @Test
+    public void testGetRelativePath()
+    {
+        ConfigSource conf = config();
+        String expected = "/path/to/sample.csv";
+
+        conf.set("password", "ABCDE");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        String uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample.csv");
+        assertEquals(expected, SftpFileInput.getRelativePath(task, Optional.of(uri)));
+
+        conf.set("password", "ABCD#$¥!%'\"@?<>\\&/_^~|-=+-,{}[]()");
+        task = config.loadConfig(PluginTask.class);
+        uri = SftpFileInput.getSftpFileUri(task, "/path/to/sample.csv");
+        assertEquals(expected, SftpFileInput.getRelativePath(task, Optional.of(uri)));
+    }
+
     private SshServer createSshServer(String host, int port, final String sshUsername, final String sshPassword)
     {
         // setup a mock sftp server

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sftp
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-19 00:00:00.000000000 Z
+date: 2016-09-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -72,7 +72,7 @@ files:
 - classpath/commons-io-1.3.2.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.1.jar
-- classpath/embulk-input-sftp-0.2.0.jar
+- classpath/embulk-input-sftp-0.2.1.jar
 - classpath/jsch-0.1.53.jar
 homepage: https://github.com/embulk/embulk-input-sftp
 licenses: