embulk-output-sftp 0.0.9 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3bd5fb09a08a951dc92e4a12745815cde65af4a4
-  data.tar.gz: ab89f5c32965abac6cc6437211b7f55bc9973d6d
+  metadata.gz: 831b167ddd205c7ad62b1306a47c634eb7c52aa5
+  data.tar.gz: 2de997335de69c887b58507cd820be9593a06483
 SHA512:
-  metadata.gz: 973356f2f106e7244e1e4865d81450146312a92df47952f435e05e982fd4ba6403b1ca78d19c4c0e79253c6080ff00f54cb843e5fb652d00836a745fe32a28f2
-  data.tar.gz: 8f2323398bd3f9c54eb26bf51b792e82198789e19c27333ade336130e296189bea54a5c970fc79a3820867d25d2407a2a739175c127c80d7341a415b830e4b6c
+  metadata.gz: cfa0bf45de009ad868e66ded095e693f53aa3a5db03d166e6460ec686668e09018106868b055bf2fbe7aaa18522beb80701de7a85768910fcbb4bfa6331860e7
+  data.tar.gz: e7ea2ee48cc39c457a57dc368ac87918697aa85c1e372fe64a7740cbaccc360ec0ce10814bad6b7029bf51799b610b7d2c441a26a38edf7b8aeb141f35bee1d1

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+0.1.1 (2017-05-29)
+- Fix: Improve retry logic
+  - https://github.com/embulk/embulk-output-sftp/pull/34
+
 0.0.9 (2017-03-09)
 ==================
 - Fix: Hide password in the log

data/README.md

@@ -1,6 +1,5 @@
-# Sftp file output plugin for Embulk
-[![Build Status](https://travis-ci.org/civitaspo/embulk-output-sftp.svg)](https://travis-ci.org/civitaspo/embulk-output-sftp)
-[![Coverage Status](https://coveralls.io/repos/civitaspo/embulk-output-sftp/badge.svg?branch=master&service=github)](https://coveralls.io/github/civitaspo/embulk-output-sftp?branch=master)
+# SFTP file output plugin for Embulk
+[![Build Status](https://travis-ci.org/embulk/embulk-output-sftp.svg)](https://travis-ci.org/embulk/embulk-output-sftp)
 
 Stores files on a SFTP Server
 
@@ -125,6 +124,7 @@ $ embulk run -Ilib example/sample.yml
 
 ```
 $ ./gradlew gem  # -t to watch change of files and rebuild continuously
+$ ./gradlew bintrayUpload # release embulk-output-sftp to Bintray maven repo
 ```
 
 ## Note

data/build.gradle

@@ -1,5 +1,6 @@
 plugins {
-    id "com.jfrog.bintray" version "1.1"
+    id "com.jfrog.bintray" version "1.7"
+    id "maven-publish"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "com.github.kt3k.coveralls" version "2.4.0"
     id "jacoco"
@@ -15,7 +16,8 @@ configurations {
     provided
 }
 
-version = "0.0.9"
+group = "org.embulk.output.sftp"
+version = "0.1.1"
 sourceCompatibility = 1.7
 targetCompatibility = 1.7
 
@@ -24,6 +26,7 @@ dependencies {
     provided "org.embulk:embulk-core:0.8.6"
     // compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
     compile "org.apache.commons:commons-vfs2:2.1.1660580.2"
+    compile "commons-io:commons-io:2.5"
     compile "com.jcraft:jsch:0.1.53"
     testCompile "junit:junit:4.+"
     testCompile "org.embulk:embulk-core:0.8.6:tests"
@@ -40,6 +43,50 @@ jacocoTestReport {
     }
 }
 
+javadoc {
+    options {
+        locale = 'en_US'
+        encoding = 'UTF-8'
+    }
+}
+
+// bintray
+bintray {
+    // write at your bintray user name and api key to ~/.gradle/gradle.properties file:
+    user = project.hasProperty('bintray_user') ? bintray_user : ''
+    key = project.hasProperty('bintray_api_key') ? bintray_api_key : ''
+
+    publications = ['bintrayMavenRelease']
+    publish = true
+
+    pkg {
+        userOrg = 'embulk-output-sftp'
+        repo = 'maven'
+        name = project.name
+        desc = 'SFTP output plugin for Embulk'
+        websiteUrl = 'https://github.com/embulk/embulk-output-sftp'
+        issueTrackerUrl = 'https://github.com/embulk/embulk-output-sftp/issues'
+        vcsUrl = 'https://github.com/embulk/embulk-output-sftp.git'
+        licenses = ['MIT']
+        labels = ['embulk', 'java']
+        publicDownloadNumbers = true
+
+        version {
+            name = project.version
+        }
+    }
+}
+publishing {
+    publications {
+        bintrayMavenRelease(MavenPublication) {
+            from components.java
+            artifact testsJar
+            artifact sourcesJar
+            artifact javadocJar
+        }
+    }
+}
+
 task classpath(type: Copy, dependsOn: ["jar"]) {
     doFirst { file("classpath").deleteDir() }
     from (configurations.runtime - configurations.provided + files(jar.archivePath))
@@ -47,6 +94,20 @@ task classpath(type: Copy, dependsOn: ["jar"]) {
 }
 clean { delete "classpath" }
 
+// add tests/javadoc/source jar tasks as artifacts to be released
+task testsJar(type: Jar, dependsOn: classes) {
+    classifier = 'tests'
+    from sourceSets.test.output
+}
+task sourcesJar(type: Jar, dependsOn: classes) {
+    classifier = 'sources'
+    from sourceSets.main.allSource
+}
+task javadocJar(type: Jar, dependsOn: javadoc) {
+    classifier = 'javadoc'
+    from javadoc.destinationDir
+}
+
 checkstyle {
     configFile = file("${project.rootDir}/config/checkstyle/checkstyle.xml")
     toolVersion = '6.14.1'
@@ -88,12 +149,12 @@ task gemspec {
 Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
-  spec.authors       = ["Civitaspo"]
-  spec.summary       = %[Sftp file output plugin for Embulk]
-  spec.description   = %[Stores files on Sftp.]
-  spec.email         = ["civitaspo@gmail.com"]
+  spec.authors       = ["Civitaspo", "Satoshi Akama"]
+  spec.summary       = %[SFTP file output plugin for Embulk]
+  spec.description   = %[Stores files on SFTP server.]
+  spec.email         = ["civitaspo@gmail.com", "satoshiakama@gmail.com"]
   spec.licenses      = ["MIT"]
-  spec.homepage      = "https://github.com/civitaspo/embulk-output-sftp"
+  spec.homepage      = "https://github.com/embulk/embulk-output-sftp"
 
   spec.files         = `git ls-files`.split("\n") + Dir["classpath/*.jar"]
   spec.test_files    = spec.files.grep(%r"^(test|spec)/")

data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java

@@ -2,6 +2,7 @@ package org.embulk.output.sftp;
 
 import com.google.common.base.Function;
 import com.google.common.base.Throwables;
+import org.apache.commons.io.IOUtils;
 import org.apache.commons.vfs2.FileObject;
 import org.apache.commons.vfs2.FileSystemException;
 import org.apache.commons.vfs2.FileSystemOptions;
@@ -15,15 +16,22 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileOutput;
 import org.embulk.spi.TransactionalFileOutput;
 import org.embulk.spi.unit.LocalFile;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.slf4j.Logger;
 
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
 
 import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 /**
  * Created by takahiro.nakayama on 10/20/15.
@@ -44,8 +52,8 @@ public class SftpFileOutput
 
     private final int taskIndex;
     private int fileIndex = 0;
-    private FileObject currentFile;
-    private OutputStream currentFileOutputStream;
+    private File tempFile;
+    private BufferedOutputStream localOutput = null;
 
     private StandardFileSystemManager initializeStandardFileSystemManager()
     {
@@ -144,11 +152,10 @@ public class SftpFileOutput
         closeCurrentFile();
 
         try {
-            currentFile = newSftpFile(getSftpFileUri(getOutputFilePath()));
-            currentFileOutputStream = newSftpOutputStream(currentFile);
-            logger.info("new sftp file: {}", currentFile.getPublicURIString());
+            tempFile = Exec.getTempFileSpace().createTempFile();
+            localOutput = new BufferedOutputStream(new FileOutputStream(tempFile));
         }
-        catch (FileSystemException e) {
+        catch (FileNotFoundException e) {
             logger.error(e.getMessage());
             Throwables.propagate(e);
         }
@@ -157,28 +164,11 @@ public class SftpFileOutput
     @Override
     public void add(final Buffer buffer)
     {
-        if (currentFile == null) {
-            throw new IllegalStateException("nextFile() must be called before poll()");
-        }
-
         try {
-            Retriable<Void> retriable = new Retriable<Void>() {
-                public Void execute() throws IOException
-                {
-                    currentFileOutputStream.write(buffer.array(), buffer.offset(), buffer.limit());
-                    return null;
-                }
-            };
-            try {
-                withConnectionRetry(retriable);
-            }
-            catch (Exception e) {
-                throw (IOException) e;
-            }
+            localOutput.write(buffer.array(), buffer.offset(), buffer.limit());
         }
-        catch (IOException e) {
-            logger.error(e.getMessage());
-            Throwables.propagate(e);
+        catch (IOException ex) {
+            throw Throwables.propagate(ex);
         }
         finally {
             buffer.release();
@@ -189,6 +179,7 @@ public class SftpFileOutput
     public void finish()
     {
         closeCurrentFile();
+        uploadFile(getOutputFilePath());
     }
 
     @Override
@@ -211,27 +202,68 @@ public class SftpFileOutput
 
     private void closeCurrentFile()
     {
-        if (currentFile == null) {
-            return;
+        if (localOutput != null) {
+            try {
+                localOutput.close();
+            }
+            catch (IOException ex) {
+                throw Throwables.propagate(ex);
+            }
         }
+    }
 
+    private Void uploadFile(final String remotePath)
+    {
         try {
-            currentFileOutputStream.close();
-        }
-        catch (IOException e) {
-            logger.info(e.getMessage());
+            return retryExecutor()
+                    .withRetryLimit(maxConnectionRetry)
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<Void>() {
+                        @Override
+                        public Void call() throws IOException
+                        {
+                            FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
+                            logger.info("new sftp file: {}", remoteFile.getPublicURIString());
+                            try (BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream())) {
+                                try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
+                                    IOUtils.copy(inputStream, outputStream);
+                                }
+                            }
+                            return null;
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
+                        {
+                            String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                logger.warn(message, exception);
+                            }
+                            else {
+                                logger.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
+                        {
+                        }
+                    });
         }
-
-        try {
-            currentFile.close();
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex.getCause());
         }
-        catch (FileSystemException e) {
-            logger.warn(e.getMessage());
+        catch (InterruptedException ex) {
+            throw Throwables.propagate(ex);
         }
-
-        fileIndex++;
-        currentFile = null;
-        currentFileOutputStream = null;
     }
 
     private URI getSftpFileUri(String remoteFilePath)
@@ -250,84 +282,20 @@ public class SftpFileOutput
         return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
     }
 
-    interface Retriable<T>
-    {
-        /**
-         * Execute the operation with the given (or null) return value.
-         * @return any return value from the operation
-         * @throws Exception
-         */
-        public T execute() throws Exception;
-    }
-
-    private <T> T withConnectionRetry(final Retriable<T> op)
-            throws Exception
+    private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
     {
-        int count = 0;
-        while (true) {
-            try {
-                return op.execute();
-            }
-            catch (final Exception e) {
-                if (++count > maxConnectionRetry) {
-                    throw e;
-                }
-                logger.warn("failed to connect sftp server: " + e.getMessage(), e);
-
-                try {
-                    long sleepTime = ((long) Math.pow(2, count) * 1000);
-                    logger.warn("sleep in next connection retry: {} milliseconds", sleepTime);
-                    Thread.sleep(sleepTime); // milliseconds
-                }
-                catch (InterruptedException e1) {
-                    // Ignore this exception because this exception is just about `sleep`.
-                    logger.warn(e1.getMessage(), e1);
-                }
-                logger.warn("retry to connect sftp server: " + count + " times");
-            }
+        FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
+        if (file.exists()) {
+            file.delete();
         }
-    }
-
-    private FileObject newSftpFile(final URI sftpUri)
-            throws FileSystemException
-    {
-        Retriable<FileObject> retriable = new Retriable<FileObject>() {
-            public FileObject execute() throws FileSystemException
-            {
-                FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
-                if (file.getParent().exists()) {
-                    logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
-                }
-                else {
-                    logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
-                    file.getParent().createFolder();
-                }
-                return file;
-            }
-        };
-        try {
-            return withConnectionRetry(retriable);
-        }
-        catch (Exception e) {
-            throw (FileSystemException) e;
-        }
-    }
-
-    private OutputStream newSftpOutputStream(final FileObject file)
-            throws FileSystemException
-    {
-        Retriable<OutputStream> retriable = new Retriable<OutputStream>() {
-            public OutputStream execute() throws FileSystemException
-            {
-                return file.getContent().getOutputStream();
-            }
-        };
-        try {
-            return withConnectionRetry(retriable);
+        if (file.getParent().exists()) {
+            logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
         }
-        catch (Exception e) {
-            throw (FileSystemException) e;
+        else {
+            logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
+            file.getParent().createFolder();
         }
+        return file;
     }
 
     private Function<LocalFile, String> localFileToPathString()

metadata

@@ -1,46 +1,48 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-sftp
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Civitaspo
+- Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-03-09 00:00:00.000000000 Z
+date: 2017-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
-  name: bundler
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  prerelease: false
+  type: :development
 - !ruby/object:Gem::Dependency
-  requirement: !ruby/object:Gem::Requirement
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-  name: rake
-  prerelease: false
-  type: :development
-  version_requirements: !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
-description: Stores files on Sftp.
+  prerelease: false
+  type: :development
+description: Stores files on SFTP server.
 email:
 - civitaspo@gmail.com
+- satoshiakama@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -67,11 +69,12 @@ files:
 - src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
 - src/test/resources/id_rsa
 - src/test/resources/id_rsa.pub
+- classpath/commons-io-2.5.jar
 - classpath/commons-logging-1.2.jar
 - classpath/commons-vfs2-2.1.1660580.2.jar
-- classpath/embulk-output-sftp-0.0.9.jar
+- classpath/embulk-output-sftp-0.1.1.jar
 - classpath/jsch-0.1.53.jar
-homepage: https://github.com/civitaspo/embulk-output-sftp
+homepage: https://github.com/embulk/embulk-output-sftp
 licenses:
 - MIT
 metadata: {}
@@ -94,5 +97,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Sftp file output plugin for Embulk
+summary: SFTP file output plugin for Embulk
 test_files: []