embulk-output-sftp 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2b11bf7261d43608476053f62b5425f45643c1a
-  data.tar.gz: dc84dd6e807ae8d0b6736eaa485ced869e8c83c2
+  metadata.gz: 3d9834eb7ff92d71bfd153216229ad010c7449d4
+  data.tar.gz: ab85abd9efaca6457c57e1349c0b0494f8a769e1
 SHA512:
-  metadata.gz: 0025aef591b2cdb982df7d09a5b2225f1855b2e7ab97966d34cd61b74eaa1e1dcab848af0881a5f4a2159d5d6d9d39f99fd064a0ed9cf3cc686a4e9359b398e0
-  data.tar.gz: 5bb8ab885d48fb2e3a4af7a5db9e850d7c185e0d2d4de8897abaf5c2580fba1ec4e16c3a5c60be91239cb4db280420f1dcdb8f863bc0132e0ac0ba0a8f62ce20
+  metadata.gz: 28fe14dde5b881475e5145272343ca6582072a29233c06871d4d24a94333c414ae1aed1643749d46c22bfc7c585dedc917d1714f11b35dfe8db8619ad08a00e5
+  data.tar.gz: 029de058c1e7167329c3d4bcfa95fd58d9a2b7e26319cf3365a91f4a7563999697d2476b91858fa8557d9362068484760e2c269e6eea05d7405bb5d44d858ef0

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+0.1.5 (2018-03-13)
+- Fix: Fix random hanging and log transfer progress
+  - https://github.com/embulk/embulk-output-sftp/pull/45
+
 0.1.4 (2017-12-21)
 - Fix: Disable remote temporary file rename logic
   - https://github.com/embulk/embulk-output-sftp/pull/41

data/build.gradle

@@ -17,7 +17,7 @@ configurations {
 }
 
 group = "org.embulk.output.sftp"
-version = "0.1.4"
+version = "0.1.5"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/src/main/java/org/embulk/output/sftp/SftpUtils.java

@@ -2,7 +2,6 @@ package org.embulk.output.sftp;
 
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
@@ -16,11 +15,11 @@ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 
-import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 
@@ -143,12 +142,32 @@ public class SftpUtils
                         @Override
                         public Void call() throws IOException
                         {
+                            long size = localTempFile.length();
+                            int step = 10; // 10% each step
+                            long bytesPerStep = size / step;
+                            long startTime = System.nanoTime();
+
                             try (FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
-                                 BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
-                                 BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localTempFile))
+                                    InputStream inputStream = new FileInputStream(localTempFile);
+                                    BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
                             ) {
-                                logger.info("new sftp file: {}", remoteFile.getPublicURIString());
-                                IOUtils.copy(inputStream, outputStream);
+                                logger.info("Uploading to remote sftp file ({} KB): {}", size / 1024, remoteFile.getPublicURIString());
+                                byte[] buffer = new byte[8 * 1024]; // 8KB ~ default buffer size of BufferedOutputStream
+                                int len = inputStream.read(buffer);
+                                long total = 0;
+                                int progress = 0;
+                                while (len != -1) {
+                                    outputStream.write(buffer, 0, len);
+                                    len = inputStream.read(buffer);
+                                    total += len;
+                                    if (total / bytesPerStep > progress) {
+                                        progress = (int) (total / bytesPerStep);
+                                        long transferRate = (long) (total / ((System.nanoTime() - startTime) / 1e9));
+                                        logger.info("Upload progress: {}% - {} KB - {} KB/s",
+                                                progress * step, total / 1024, transferRate / 1024);
+                                    }
+                                }
+                                logger.info("Upload completed.");
                             }
                             return null;
                         }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Civitaspo
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-12-21 00:00:00.000000000 Z
+date: 2018-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,7 +73,7 @@ files:
 - classpath/commons-io-2.6.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.2.jar
-- classpath/embulk-output-sftp-0.1.4.jar
+- classpath/embulk-output-sftp-0.1.5.jar
 - classpath/jsch-0.1.54.jar
 homepage: https://github.com/embulk/embulk-output-sftp
 licenses: