embulk-output-sftp 0.1.11 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 98edf71b4a5d7998403671b7e181b250f7db9981
-  data.tar.gz: 940f0d1fea8ed83c957bdfca08a2bb89725b487b
+  metadata.gz: 3140b0cc46176332801c67fbf286548785e5c5a3
+  data.tar.gz: a4fa818bcc4970eaf17e6a8b255ff06e35d605de
 SHA512:
-  metadata.gz: c4e8426ac217df25329da8a514b1603158bc7d6ed2b630b2e28486fcd36fb81e24aee588e5b1b2f6dea2e0996789a947eaac617330cb6180ca3e6afc35a9f0ed
-  data.tar.gz: 99c6f36a658a92dd3710c9b305892e5961f9d154694570b5f05e7cbf6e4ad2794adfda169e175622f3bb7ff5aa5714e25368e41a0f6224d3dce8acc004b87d9d
+  metadata.gz: e4a07a88cf20aebd20e14302d2f504809d509b630ed7c7dfc0e2de48d8d5f51d6b7172e8f9b86540a24dca1f1f2e4f32da92005b43f2a5fa88c3a5c55a390469
+  data.tar.gz: c71a56791aac8f6a68b6de286af446651e60719fe11996fa91d7a95ae356de87f1814dc068c84df7bb319d77262d6438f46d5e68163c1498a2c5dd62b7f461b8

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+0.2.0 (2018-08-29)
+- Fix bug of stalling when closing `remoteFile`, this version is recommended over 0.1.11
+  - https://github.com/embulk/embulk-output-sftp/pull/51
 0.1.11 (2018-08-27)
 - Enhance: Add 2 new configs `local_temp_file` (boolean) and `temp_file_threshold` (long)
   - https://github.com/embulk/embulk-output-sftp/pull/50

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 }
 
 group = "org.embulk.output.sftp"
-version = "0.1.11"
+version = "0.2.0"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/output/sftp/SftpUtils.java

@@ -20,6 +20,7 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 
 import java.io.BufferedOutputStream;
+import java.io.Closeable;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -39,7 +40,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 public class SftpUtils
 {
     private final Logger logger = Exec.getLogger(SftpUtils.class);
-    private final DefaultFileSystemManager manager;
+    private DefaultFileSystemManager manager;
     private final FileSystemOptions fsOptions;
     private final String userInfo;
     private final String user;
@@ -159,15 +160,24 @@ public class SftpUtils
                 final FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
                 final BufferedOutputStream outputStream = openStream(remoteFile);
                 // When channel is broken, closing resource may hang, hence the time-out wrapper
-                // Note: closing FileObject will also close OutputStream
-                try (TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
+                try (final TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
                     appendFile(localTempFile, remoteFile, outputStream);
                     return null;
                 }
                 finally {
-                    remoteFile.close();
+                    // closing sequentially
+                    new TimeoutCloser(remoteFile).close();
                 }
             }
+
+            @Override
+            public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+            {
+                super.onRetry(exception, retryCount, retryLimit, retryWait);
+                // re-connect
+                manager.close();
+                manager = initializeStandardFileSystemManager();
+            }
         });
     }
 
@@ -258,18 +268,9 @@ public class SftpUtils
         return manager.resolveFile(getSftpFileUri(remoteFilePath).toString(), fsOptions);
     }
 
-    BufferedOutputStream openStream(final FileObject remoteFile)
+    BufferedOutputStream openStream(final FileObject remoteFile) throws FileSystemException
     {
-        // output stream is already a BufferedOutputStream, no need to wrap
-        final String taskName = "SFTP open stream";
-        return withRetry(new DefaultRetry<BufferedOutputStream>(taskName)
-        {
-            @Override
-            public BufferedOutputStream call() throws Exception
-            {
-                return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
-            }
-        });
+        return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
     }
 
     URI getSftpFileUri(String remoteFilePath)

data/src/main/java/org/embulk/output/sftp/utils/TimeoutCloser.java

@@ -10,8 +10,7 @@ import java.util.concurrent.TimeoutException;
 
 public class TimeoutCloser implements Closeable
 {
-    @VisibleForTesting
-    int timeout = 300; // 5 minutes
+    private static int timeout = 300; // 5 minutes
     private Closeable wrapped;
 
     public TimeoutCloser(Closeable wrapped)
@@ -39,4 +38,10 @@ public class TimeoutCloser implements Closeable
             throw Throwables.propagate(e);
         }
     }
+
+    @VisibleForTesting
+    public static void setTimeout(int timeout)
+    {
+        TimeoutCloser.timeout = timeout;
+    }
 }

data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java

@@ -768,7 +768,7 @@ public class TestSftpFileOutputPlugin
     }
 
     @Test
-    public void testOpenStreamWithRetry() throws FileSystemException
+    public void testOpenStreamWithoutRetry() throws FileSystemException
     {
         SftpFileOutputPlugin.PluginTask task = defaultTask();
         SftpUtils utils = new SftpUtils(task);
@@ -778,9 +778,13 @@ public class TestSftpFileOutputPlugin
                 .doCallRealMethod()
                 .when(mock).getContent();
 
-        OutputStream stream = utils.openStream(mock);
-        assertNotNull(stream);
-        Mockito.verify(mock, Mockito.times(2)).getContent();
+        try {
+            utils.openStream(mock);
+            fail("Should not reach here");
+        }
+        catch (FileSystemException e) {
+            Mockito.verify(mock, Mockito.times(1)).getContent();
+        }
     }
 
     @Test

data/src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java

@@ -32,7 +32,7 @@ public class TestTimeoutCloser
                 }
             }
         });
-        closer.timeout = 1;
+        TimeoutCloser.setTimeout(1);
         try {
             closer.close();
             fail("Should not finish");

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.2.0
 platform: ruby
 authors:
 - Civitaspo
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-08-27 00:00:00.000000000 Z
+date: 2018-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +83,7 @@ files:
 - src/test/resources/id_rsa.pub
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.2.jar
-- classpath/embulk-output-sftp-0.1.11.jar
+- classpath/embulk-output-sftp-0.2.0.jar
 - classpath/commons-io-2.6.jar
 - classpath/jsch-0.1.54.jar
 homepage: https://github.com/embulk/embulk-output-sftp