embulk-output-sftp 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 865291280d7c6936d0e4f12655a00b854f880153
-  data.tar.gz: dfe086b9210d16e7da2e21d5f77e42c38bd7f23b
+  metadata.gz: f5c4bbee5ba87ccfee03e1423e182ea29279fd33
+  data.tar.gz: 04c0f1a8048c4240ed7156195e8a3e307db4fd92
 SHA512:
-  metadata.gz: b226b3a68c6e81ddf17ff22577ffd80cae2b58201b18a38bae681b0729e531769426e3f760694d36121e1885a6dfc21383a91c4fe5664203aafb58ad8c00c0fd
-  data.tar.gz: 6bdda04f1ba31689656896323098be6e43e34c88cec3ea7922c6e0c55c6d8bc3b593f89ea863fcefb8a537da2ae38e6a2ebf159008e44e5d475cc025014fe80d
+  metadata.gz: e550bcf13c15c094de8d157fccaec8e0fc33eb04b963ced46e7ad68a6140816cf0b64389607e8929efabdd8c5ff8fe2b41f6d84be1946a5c6d776a889f09c5e1
+  data.tar.gz: 85d0eefa677889b005c6d50f719b185e7d0a32269518b541326135ad18b5d35c044a72daa6929c0953f96472820cb938ff99e053a2e02f4f869fc2fe3d03739f

data/.gitignore

@@ -8,3 +8,4 @@ build/
 .idea
 *.iml
 example/config.yml
+.vagrant

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+0.0.6 (2016-03-16)
+==================
+- Fix: Avoid some connection errors
+  - https://github.com/civitaspo/embulk-output-sftp/pull/17
+
 0.0.5 (2016-03-09)
 ==================
 - Add: Support MapReduce executor

data/README.md

@@ -103,10 +103,21 @@ out:
 ```
 
 ## Run Example
-replace settings in `example/sample.yml` before running.
 
 ```
 $ ./gradlew classpath
+```
+
+Use `vagrant` to start a remote sshd server:
+
+```
+$ vagrant up
+```
+
+Run:
+
+
+```
 $ embulk run -Ilib example/sample.yml
 ```
 
@@ -123,3 +134,4 @@ This plugin uses "org.apache.commons:commons-vfs" and the library uses the logge
 ## Contributors
 - Satoshi Akama (@sakama)
 - Rudolph Miller (@Rudolph-Miller)
+- Naotoshi Seo (@sonots)

data/Vagrantfile

@@ -0,0 +1,20 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Every Vagrant virtual environment requires a box to build off of.
+  config.vm.box = "centos6.5.3"
+  config.vm.box_url = 'https://github.com/2creatives/vagrant-centos/releases/download/v6.5.3/centos65-x86_64-20140116.box'
+
+  # name
+  config.vm.define "vagrant-centos"
+  config.ssh.forward_agent = true
+  config.vm.network "forwarded_port", guest: 22, host: 2210
+end

data/build.gradle

@@ -15,7 +15,7 @@ configurations {
     provided
 }
 
-version = "0.0.5"
+version = "0.0.6"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 

data/example/{sample.yml → sample.yml.liquid}

@@ -16,11 +16,11 @@ in:
 out:
   type: sftp
   host: 127.0.0.1
-  port: 22
-  user: your_name
-  password: your_password
-  secret_key_file: your_secret_key_file
-  secret_key_passphrase: your_secret_key_passphrase
+  port: 2210
+  user: vagrant
+  # password:
+  secret_key_file: {{ env.PWD }}/.vagrant/machines/vagrant-centos/virtualbox/private_key
+  # secret_key_passphrase:
   user_directory_is_root: true
   path_prefix: /tmp/embulk_output_sftp/data
   file_ext: .tsv
@@ -36,4 +36,5 @@ out:
       quote: "\""
       escape: "\\"
       null_string: ""
-      default_timezone: 'UTC'
+      default_timezone: 'UTC'
+

data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java

@@ -17,6 +17,7 @@ import org.embulk.spi.TransactionalFileOutput;
 import org.embulk.spi.unit.LocalFile;
 import org.slf4j.Logger;
 
+import java.lang.Void;
 import java.io.File;
 import java.io.IOException;
 import java.io.OutputStream;
@@ -144,7 +145,7 @@ public class SftpFileOutput
 
         try {
             currentFile = newSftpFile(getSftpFileUri(getOutputFilePath()));
-            currentFileOutputStream = currentFile.getContent().getOutputStream();
+            currentFileOutputStream = newSftpOutputStream(currentFile);
             logger.info("new sftp file: {}", currentFile.getPublicURIString());
         }
         catch (FileSystemException e) {
@@ -154,14 +155,26 @@ public class SftpFileOutput
     }
 
     @Override
-    public void add(Buffer buffer)
+    public void add(final Buffer buffer)
     {
         if (currentFile == null) {
             throw new IllegalStateException("nextFile() must be called before poll()");
         }
 
         try {
-            currentFileOutputStream.write(buffer.array(), buffer.offset(), buffer.limit());
+            Retriable<Void> retriable = new Retriable<Void>() {
+                public Void execute() throws IOException
+                {
+                    currentFileOutputStream.write(buffer.array(), buffer.offset(), buffer.limit());
+                    return null;
+                }
+            };
+            try {
+                withConnectionRetry(retriable);
+            }
+            catch (Exception e) {
+                throw (IOException)e;
+            }
         }
         catch (IOException e) {
             logger.error(e.getMessage());
@@ -204,18 +217,21 @@ public class SftpFileOutput
 
         try {
             currentFileOutputStream.close();
-            currentFile.getContent().close();
-            currentFile.close();
         }
         catch (IOException e) {
-            logger.error(e.getMessage());
-            Throwables.propagate(e);
+            logger.info(e.getMessage());
         }
-        finally {
-            fileIndex++;
-            currentFile = null;
-            currentFileOutputStream = null;
+
+        try {
+            currentFile.close();
         }
+        catch (FileSystemException e) {
+            logger.warn(e.getMessage());
+        }
+
+        fileIndex++;
+        currentFile = null;
+        currentFileOutputStream = null;
     }
 
     private URI getSftpFileUri(String remoteFilePath)
@@ -234,24 +250,25 @@ public class SftpFileOutput
         return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
     }
 
-    private FileObject newSftpFile(URI sftpUri)
-            throws FileSystemException
+    interface Retriable<T>
     {
+        /**
+         * Execute the operation with the given (or null) return value.
+         * 
+         * @return any return value from the operation
+         * @throws Exception
+         */
+        public T execute() throws Exception;
+    }
+
+    private <T> T withConnectionRetry( final Retriable<T> op ) throws Exception {
         int count = 0;
         while (true) {
             try {
-                FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
-                if (file.getParent().exists()) {
-                    logger.info("parent directory {} exists there", file.getParent());
-                    return file;
-                }
-                else {
-                    logger.info("trying to create parent directory {}", file.getParent());
-                    file.getParent().createFolder();
-                }
+                return op.execute();
             }
-            catch (FileSystemException e) {
-                if (++count == maxConnectionRetry) {
+            catch(final Exception e) {
+                if (++count > maxConnectionRetry) {
                     throw e;
                 }
                 logger.warn("failed to connect sftp server: " + e.getMessage(), e);
@@ -270,6 +287,48 @@ public class SftpFileOutput
         }
     }
 
+    private FileObject newSftpFile(final URI sftpUri)
+            throws FileSystemException
+    {
+        Retriable<FileObject> retriable = new Retriable<FileObject>() {
+            public FileObject execute() throws FileSystemException
+            {
+                FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
+                if (file.getParent().exists()) {
+                    logger.info("parent directory {} exists there", file.getParent());
+                }
+                else {
+                    logger.info("trying to create parent directory {}", file.getParent());
+                    file.getParent().createFolder();
+                }
+                return file;
+            }
+        };
+        try {
+            return withConnectionRetry(retriable);
+        }
+        catch (Exception e) {
+            throw (FileSystemException)e;
+        }
+    }
+
+    private OutputStream newSftpOutputStream(final FileObject file)
+            throws FileSystemException
+    {
+        Retriable<OutputStream> retriable = new Retriable<OutputStream>() {
+            public OutputStream execute() throws FileSystemException
+            {
+                return file.getContent().getOutputStream();
+            }
+        };
+        try {
+            return withConnectionRetry(retriable);
+        }
+        catch (Exception e) {
+            throw (FileSystemException)e;
+        }
+    }
+
     private Function<LocalFile, String> localFileToPathString()
     {
         return new Function<LocalFile, String>()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Civitaspo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-03-09 00:00:00.000000000 Z
+date: 2016-03-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -50,11 +50,12 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- Vagrantfile
 - build.gradle
 - config/checkstyle/checkstyle.xml
 - config/checkstyle/default.xml
 - example/data.csv
-- example/sample.yml
+- example/sample.yml.liquid
 - gradle/wrapper/gradle-wrapper.jar
 - gradle/wrapper/gradle-wrapper.properties
 - gradlew
@@ -68,7 +69,7 @@ files:
 - src/test/resources/id_rsa.pub
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.1.1660580.2.jar
-- classpath/embulk-output-sftp-0.0.5.jar
+- classpath/embulk-output-sftp-0.0.6.jar
 - classpath/jsch-0.1.53.jar
 homepage: https://github.com/civitaspo/embulk-output-sftp
 licenses: