embulk-input-sftp 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 66e8dab8cd68fdd0a9844dac72d51eaf070cf93a
-  data.tar.gz: 9de2db631fa24c526e448a2985003063f05a2ae7
+  metadata.gz: 107c7dbf7c423b83b5677db6da16b272c8dccab4
+  data.tar.gz: edcada631a85858a63c57b7663aedbdb2ab365f6
 SHA512:
-  metadata.gz: 5079bf38ede9c5f4b611831144cdcc307c3b71acc6f2b2e43f3570d67abf21f4ca762dddfc0273b3be6cb7740072beaad0865fd5e4a98360d8206c1c88cfe3b0
-  data.tar.gz: 538c81c868addc2d141770ab5c88d3d38aa0e9e297214a2a42255e80f4f7de91da7ac646eae6aababe301b611ace77df261e7a43931e98c4d3e76bb0100c9e02
+  metadata.gz: d28c5e3ba7db475342b75df902b2f588e3259384138e3b1ee696e6582aa1d9ea1fecf5e2f0955cda3a2e19454b69c1d1d346c53b1153030b6406e0680cf03519
+  data.tar.gz: 1e7b0e362480f2e610c87871f49f020458054ea5d114a6ab064dde3bf32cf021e44a1954a1ada5f7057fe85d4e780ceaec9b7c5c0e277af2d4f99b283646f26d

data/CHANGELOG.md

@@ -1,6 +1,10 @@
+## 0.1.2 - 2015-03-23
+
+* [maintenance] Use RetryExecutor when retrying that is provide by embulk-core [#9](https://github.com/sakama/embulk-input-sftp/pull/9)
+
 ## 0.1.1 - 2015-03-18
 
 * [feature] Support last_path_ option [#2](https://github.com/sakama/embulk-input-sftp/pull/2)[#4](https://github.com/sakama/embulk-input-sftp/pull/4)[#7](https://github.com/sakama/embulk-input-sftp/pull/7)
 * [feature] Support path_match_pattern option [#6](https://github.com/sakama/embulk-input-sftp/pull/6)
 * [maintenance] Add unit test [#3](https://github.com/sakama/embulk-input-sftp/pull/3)
-* [maintenance] Skip retry of file downloading when permission denied error happens [#1](https://github.com/sakama/embulk-input-sftp/pull/1)
+* [maintenance] Skip retry of file downloading when permission denied error happens [#1](https://github.com/sakama/embulk-input-sftp/pull/1)

data/README.md

@@ -21,8 +21,6 @@ Reads files stored on remote server using SFTP
 - **path_prefix**: Prefix of output paths (string, required)
 - **path_match_pattern**: regexp to match file paths. If a file path doesn't match with this pattern, the file will be skipped (regexp string, optional)
 - **total_file_count_limit**: maximum number of files to read (integer, optional)
-- **file_ext**: Extension of output files (string, required)
-- **sequence_format**: Format for sequence part of output files (string, default: `".%03d.%02d"`)
 - **min_task_size (experimental)**: minimum size of a task. If this is larger than 0, one task includes multiple input files. This is useful if too many number of tasks impacts performance of output or executor plugins badly. (integer, optional)
 
 ### Proxy configuration

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/input/sftp/SftpFileInput.java

@@ -16,9 +16,13 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.unit.LocalFile;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -56,6 +60,7 @@ public class SftpFileInput
         }
 
         StandardFileSystemManager manager = new StandardFileSystemManager();
+        manager.setClassLoader(SftpFileInput.class.getClassLoader());
         try {
             manager.init();
         }
@@ -149,64 +154,81 @@ public class SftpFileInput
         }
     }
 
-    public static FileList listFilesByPrefix(PluginTask task)
+    public static FileList listFilesByPrefix(final PluginTask task)
     {
-        FileList.Builder builder = new FileList.Builder(task);
+        final FileList.Builder builder = new FileList.Builder(task);
         int maxConnectionRetry = task.getMaxConnectionRetry();
-        String lastKey = null;
 
-        StandardFileSystemManager manager = null;
-        int count = 0;
-        while (true) {
-            try {
-                manager = initializeStandardFileSystemManager();
-                FileSystemOptions fsOptions = initializeFsOptions(task);
+        try {
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<FileList>() {
+                        @Override
+                        public FileList call() throws IOException
+                        {
+                            String lastKey = null;
+                            log.info("Getting to download file list");
+                            StandardFileSystemManager manager = initializeStandardFileSystemManager();
+                            FileSystemOptions fsOptions = initializeFsOptions(task);
 
-                if (task.getLastPath().isPresent() && !task.getLastPath().get().isEmpty()) {
-                    lastKey = manager.resolveFile(getSftpFileUri(task, task.getLastPath().get()), fsOptions).toString();
-                }
+                            if (task.getLastPath().isPresent() && !task.getLastPath().get().isEmpty()) {
+                                lastKey = manager.resolveFile(getSftpFileUri(task, task.getLastPath().get()), fsOptions).toString();
+                            }
 
-                FileObject files = manager.resolveFile(getSftpFileUri(task, task.getPathPrefix()), fsOptions);
-                String basename = FilenameUtils.getBaseName(task.getPathPrefix());
-                if (files.isFolder()) {
-                    for (FileObject f : files.getChildren()) {
-                        if (f.isFile()) {
-                            addFileToList(builder, f.toString(), f.getContent().getSize(), "", lastKey);
+                            FileObject files = manager.resolveFile(getSftpFileUri(task, task.getPathPrefix()), fsOptions);
+                            String basename = FilenameUtils.getBaseName(task.getPathPrefix());
+                            if (files.isFolder()) {
+                                for (FileObject f : files.getChildren()) {
+                                    if (f.isFile()) {
+                                        addFileToList(builder, f.toString(), f.getContent().getSize(), "", lastKey);
+                                    }
+                                }
+                            }
+                            else {
+                                FileObject parent = files.getParent();
+                                for (FileObject f : parent.getChildren()) {
+                                    if (f.isFile()) {
+                                        addFileToList(builder, f.toString(), f.getContent().getSize(), basename, lastKey);
+                                    }
+                                }
+                            }
+                            return builder.build();
                         }
-                    }
-                }
-                else {
-                    FileObject parent = files.getParent();
-                    for (FileObject f : parent.getChildren()) {
-                        if (f.isFile()) {
-                            addFileToList(builder, f.toString(), f.getContent().getSize(), basename, lastKey);
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;
                         }
-                    }
-                }
-                return builder.build();
-            }
-            catch (FileSystemException ex) {
-                if (++count == maxConnectionRetry) {
-                    Throwables.propagate(ex);
-                }
-                log.warn("failed to connect sftp server: " + ex.getMessage(), ex);
 
-                try {
-                    long sleepTime = ((long) Math.pow(2, count) * 1000);
-                    log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
-                    Thread.sleep(sleepTime); // milliseconds
-                }
-                catch (InterruptedException ex2) {
-                    // Ignore this exception because this exception is just about `sleep`.
-                    log.warn(ex2.getMessage(), ex2);
-                }
-                log.warn("retrying to connect sftp server: " + count + " times");
-            }
-            finally {
-                if (manager != null) {
-                    manager.close();
-                }
-            }
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            String message = String.format("SFTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex.getCause());
+        }
+        catch (InterruptedException ex) {
+            throw Throwables.propagate(ex);
         }
     }
 

data/src/main/java/org/embulk/input/sftp/SingleFileProvider.java

@@ -1,15 +1,20 @@
 package org.embulk.input.sftp;
 
+import com.google.common.base.Throwables;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
 import org.apache.commons.vfs2.impl.StandardFileSystemManager;
 import org.embulk.spi.Exec;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InterruptedIOException;
 import java.util.Iterator;
 
 public class SingleFileProvider
@@ -37,33 +42,58 @@ public class SingleFileProvider
             return null;
         }
         opened = true;
-        String key = iterator.next();
+        final String key = iterator.next();
 
-        int count = 0;
-        while (true) {
-            try {
-                FileObject file = manager.resolveFile(key, fsOptions);
-                log.info("Starting to download file {}", key);
+        try {
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<InputStream>() {
+                        @Override
+                        public InputStream call() throws FileSystemException
+                        {
+                            log.info("Starting to download file {}", key);
+                            FileObject file = manager.resolveFile(key, fsOptions);
+                            return file.getContent().getInputStream();
+                        }
 
-                return file.getContent().getInputStream();
-            }
-            catch (FileSystemException ex) {
-                if (++count == maxConnectionRetry || ex.getMessage().indexOf("Permission denied") > 0) {
-                    throw ex;
-                }
-                log.warn("failed to connect sftp server: " + ex.getMessage(), ex);
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;
+                        }
 
-                try {
-                    long sleepTime = ((long) Math.pow(2, count) * 1000);
-                    log.warn("sleep in next connection retry: {} milliseconds", sleepTime);
-                    Thread.sleep(sleepTime); // milliseconds
-                }
-                catch (InterruptedException ex2) {
-                    // Ignore this exception because this exception is just about `sleep`.
-                    log.warn(ex2.getMessage(), ex2);
-                }
-                log.warn("retrying to connect sftp server: " + count + " times");
-            }
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                                throws RetryGiveupException
+                        {
+                            if (exception.getMessage().indexOf("Permission denied") > 0) {
+                                log.error("Could not download file due to Permission Denied");
+                                throw new RetryGiveupException(exception);
+                            }
+                            String message = String.format("SFTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException)
+                                throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex.getCause());
+        }
+        catch (InterruptedException ex) {
+            throw new InterruptedIOException();
         }
     }
 

data/src/test/java/org/embulk/input/sftp/TestSftpFileInputPlugin.java

@@ -183,8 +183,8 @@ public class TestSftpFileInputPlugin
     @Test
     public void testListFiles() throws Exception
     {
-        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv");
-        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv");
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", true);
 
         PluginTask task = config.loadConfig(PluginTask.class);
 
@@ -215,8 +215,8 @@ public class TestSftpFileInputPlugin
     @Test
     public void testSftpInputByOpen() throws Exception
     {
-        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv");
-        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv");
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", true);
 
         PluginTask task = config.loadConfig(PluginTask.class);
         runner.transaction(config, new Control());
@@ -235,8 +235,8 @@ public class TestSftpFileInputPlugin
 //        try {
 //            proxyServer = createProxyServer(PROXY_PORT);
 //
-//            uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv");
-//            uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv");
+//            uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+//            uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", true);
 //
 //            ConfigSource config = Exec.newConfigSource()
 //                    .set("host", HOST)
@@ -269,8 +269,8 @@ public class TestSftpFileInputPlugin
     @Test
     public void testSftpInputByOpenTimeout() throws Exception
     {
-        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv");
-        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv");
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", true);
+        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", true);
 
         ConfigSource config = Exec.newConfigSource()
                 .set("host", HOST)
@@ -289,6 +289,29 @@ public class TestSftpFileInputPlugin
         runner.transaction(config, new Control());
     }
 
+    @Test
+    public void testSftpInputByOpenFailWithRetry() throws Exception
+    {
+        uploadFile(Resources.getResource("sample_01.csv").getPath(), REMOTE_DIRECTORY + "sample_01.csv", false);
+        uploadFile(Resources.getResource("sample_02.csv").getPath(), REMOTE_DIRECTORY + "sample_02.csv", false);
+
+        ConfigSource config = Exec.newConfigSource()
+                .set("host", HOST)
+                .set("port", PORT)
+                .set("user", USERNAME)
+                .set("password", PASSWORD)
+                .set("path_prefix", REMOTE_DIRECTORY)
+                .set("max_connection_retry", 2)
+                .set("last_path", "")
+                .set("parser", parserConfig(schemaConfig()));
+
+        exception.expect(RuntimeException.class);
+        exception.expectCause(CoreMatchers.<Throwable>instanceOf(FileSystemException.class));
+        exception.expectMessage(CoreMatchers.containsString("Unknown message with code \"java.nio.file.AccessDeniedException"));
+
+        runner.transaction(config, new Control());
+    }
+
     @Test
     public void testProxyType()
     {
@@ -377,7 +400,7 @@ public class TestSftpFileInputPlugin
                 .start();
     }
 
-    private void uploadFile(String localPath, String remotePath) throws Exception
+    private void uploadFile(String localPath, String remotePath, boolean isReadable) throws Exception
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
@@ -388,11 +411,15 @@ public class TestSftpFileInputPlugin
         while (true) {
             try {
                 StandardFileSystemManager manager = new StandardFileSystemManager();
+                manager.setClassLoader(TestSftpFileInputPlugin.class.getClassLoader());
                 manager.init();
 
                 FileObject localFile = manager.resolveFile(localPath);
                 FileObject remoteFile = manager.resolveFile(uri, fsOptions);
                 remoteFile.copyFrom(localFile, Selectors.SELECT_SELF);
+                if (!isReadable) {
+                    remoteFile.setReadable(false, false);
+                }
 
                 if (log.isDebugEnabled()) {
                     FileObject files = manager.resolveFile(SftpFileInput.getSftpFileUri(task, REMOTE_DIRECTORY));

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-18 00:00:00.000000000 Z
+date: 2016-03-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -71,7 +71,7 @@ files:
 - classpath/commons-io-1.3.2.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.1.1660580.2.jar
-- classpath/embulk-input-sftp-0.1.1.jar
+- classpath/embulk-input-sftp-0.1.2.jar
 - classpath/jsch-0.1.53.jar
 homepage: https://github.com/sakama/embulk-input-sftp
 licenses: