embulk-output-ftp 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 99f92ff252075824f860a2929aa8dc90609b3760
-  data.tar.gz: ae7284ac04e110cb17b02f5830a2bdc52c5e309c
+  metadata.gz: b0494ae651d3e5c17bb41c2cb08712ed7d6388d2
+  data.tar.gz: 9463836af3ca53940c83237a3e84126dadb8d153
 SHA512:
-  metadata.gz: 6f867e7103b201e4ddbab093001d6fa657d4932c469bae0baca717ce618b45d6e72521ffae58f0aab43c93d171250e366cf3666b204be31b7906fdbd00505c92
-  data.tar.gz: 36e2127321a3cef31515fe3b037c35db61c3187afdf5fd7010e8de2e64c9eca48f6eb1eee83f55498f79f86b7ed232d4c1e54c9e06ef487ecdd5a64e9b7d7382
+  metadata.gz: 66c737b48bc821ca444806d7d555ed17c1f7b4385f75743428a2fa3d94e72488791c0aa9883720155ac86e4d65de7e08f2c6af91afbf6ed7756cdb558116e67f
+  data.tar.gz: ccd8a59f0f4b58d9e02158ad757fd490c6e678768bf2e8f75aaa0a74dd0bfcfc465522b8126fe47710bf583c585218b860d410ccdd027c2e23ca4b92f8e41112

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.1.5 - 2016-08-08
+
+* [maintenance] Improve connection stability [#8](https://github.com/embulk/embulk-output-ftp/pull/9)
+
 ## 0.1.4 - 2016-07-29
 
 * [maintenance] Fix default port number generation logic [#7](https://github.com/embulk/embulk-output-ftp/pull/7)

data/README.md

@@ -141,7 +141,8 @@ FTP_TEST_SSL__PORT (default:990)
 
 If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
 ```xml
-$ vi ~/Library/LaunchAgents/environment.plist
+
+launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE$ vi ~/Library/LaunchAgents/environment.plist
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
@@ -157,7 +158,10 @@ $ vi ~/Library/LaunchAgents/environment.plist
       launchctl setenv FTP_TEST_USER username
       launchctl setenv FTP_TEST_PASSWORD password
       launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE /path/to/cert.pem
-      launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA /path/to/cert.pem
+      launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA "-----BEGIN CERTIFICATE-----
+      ABCDEFG...
+      EFGHIJKL...
+      -----END CERTIFICATE-----"
     </string>
   </array>
   <key>RunAtLoad</key>

data/build.gradle

@@ -14,7 +14,7 @@ configurations {
     provided
 }
 
-version = "0.1.4"
+version = "0.1.5"
 
 sourceCompatibility = 1.7
 targetCompatibility = 1.7

data/src/main/java/org/embulk/output/ftp/FtpFileOutputPlugin.java

@@ -34,6 +34,8 @@ import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.ConnectException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.List;
@@ -353,14 +355,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
             //client.setDataTimeout
             //client.setAutodetectUTF8
 
-            if (task.getPort().isPresent()) {
-                client.connect(task.getHost(), task.getPort().get());
-                log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
-            }
-            else {
-                client.connect(task.getHost());
-                log.info("Connecting to {}", task.getHost());
-            }
+            client = connect(client, task);
 
             if (task.getUser().isPresent()) {
                 log.info("Logging in with user {}", task.getUser().get());
@@ -405,6 +400,71 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
         }
     }
 
+    private static FTPClient connect(final FTPClient client, final PluginTask task) throws InterruptedIOException
+    {
+        try {
+            return retryExecutor()
+                    .withRetryLimit(task.getMaxConnectionRetry())
+                    .withInitialRetryWait(500)
+                    .withMaxRetryWait(30 * 1000)
+                    .runInterruptible(new Retryable<FTPClient>() {
+                        @Override
+                        public FTPClient call()
+                        {
+                            try {
+                                if (task.getPort().isPresent()) {
+                                    client.connect(task.getHost(), task.getPort().get());
+                                    log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
+                                }
+                                else {
+                                    client.connect(task.getHost());
+                                    log.info("Connecting to {}", task.getHost());
+                                }
+                            }
+                            catch (FTPIllegalReplyException | FTPException | IOException ex) {
+                                throw Throwables.propagate(ex);
+                            }
+                            return client;
+                        }
+
+                        @Override
+                        public boolean isRetryableException(Exception exception)
+                        {
+                            if (exception.getCause() != null) {
+                                if (exception.getCause() instanceof ConnectException && exception.getMessage().contains("Connection refused")) {
+                                    return false;
+                                }
+                            }
+                            return true;
+                        }
+
+                        @Override
+                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
+                        {
+                            String message = String.format("FTP Put request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
+                                    retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
+                            if (retryCount % 3 == 0) {
+                                log.warn(message, exception);
+                            }
+                            else {
+                                log.warn(message);
+                            }
+                        }
+
+                        @Override
+                        public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
+                        {
+                        }
+                    });
+        }
+        catch (RetryGiveupException ex) {
+            throw Throwables.propagate(ex);
+        }
+        catch (InterruptedException ex) {
+            throw new InterruptedIOException();
+        }
+    }
+
     static void disconnectClient(FTPClient client)
     {
         if (client != null && client.isConnected()) {

data/src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java

@@ -51,6 +51,7 @@ public class TestFtpFileOutputPlugin
     private static String FTP_TEST_USER;
     private static String FTP_TEST_PASSWORD;
     private static String FTP_TEST_SSL_TRUSTED_CA_CERT_FILE;
+    private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
     private static String FTP_TEST_DIRECTORY;
     private static String FTP_TEST_PATH_PREFIX;
     private static String LOCAL_PATH_PREFIX;
@@ -72,8 +73,9 @@ public class TestFtpFileOutputPlugin
         FTP_TEST_USER = System.getenv("FTP_TEST_USER");
         FTP_TEST_PASSWORD = System.getenv("FTP_TEST_PASSWORD");
         FTP_TEST_SSL_TRUSTED_CA_CERT_FILE = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_FILE");
+        FTP_TEST_SSL_TRUSTED_CA_CERT_DATA = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_DATA");
         // skip test cases, if environment variables are not set.
-        assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE);
+        assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE, FTP_TEST_SSL_TRUSTED_CA_CERT_DATA);
 
         FTP_TEST_DIRECTORY = System.getenv("FTP_TEST_DIRECTORY") != null ? getDirectory(System.getenv("FTP_TEST_DIRECTORY")) : getDirectory("/unittest/");
         FTP_TEST_PATH_PREFIX = FTP_TEST_DIRECTORY + "sample_";
@@ -149,6 +151,7 @@ public class TestFtpFileOutputPlugin
                 .set("password", FTP_TEST_PASSWORD)
                 .set("path_prefix", "my-prefix")
                 .set("file_ext", ".csv")
+                .set("max_connection_retry", 3)
                 .set("formatter", formatterConfig());
 
         Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
@@ -157,7 +160,7 @@ public class TestFtpFileOutputPlugin
     }
 
     @Test
-    public void testTransactionWithSsl()
+    public void testTransactionWithUnverifiedSsl()
     {
         ConfigSource config = Exec.newConfigSource()
                 .set("in", inputConfig())
@@ -170,6 +173,29 @@ public class TestFtpFileOutputPlugin
                 .set("ssl", true)
                 .set("ssl_verify", false)
                 .set("ssl_verify_hostname", false)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
+    @Test
+    public void testTransactionWithVerifiedSslWithCertFilePath()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", FTP_TEST_HOST)
+                .set("port", FTP_TEST_SSL_PORT)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("ssl", true)
+                .set("ssl_verify", true)
+                .set("ssl_verify_hostname", false)
                 .set("ssl_trusted_ca_cert_file", FTP_TEST_SSL_TRUSTED_CA_CERT_FILE)
                 .set("path_prefix", "my-prefix")
                 .set("file_ext", ".csv")
@@ -180,6 +206,30 @@ public class TestFtpFileOutputPlugin
         runner.transaction(config, schema, 0, new Control());
     }
 
+    @Test
+    public void testTransactionWithVerifiedSslWithCertFileData()
+    {
+        ConfigSource config = Exec.newConfigSource()
+                .set("in", inputConfig())
+                .set("parser", parserConfig(schemaConfig()))
+                .set("type", "ftp")
+                .set("host", FTP_TEST_HOST)
+                .set("port", FTP_TEST_SSL_PORT)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("ssl", true)
+                .set("ssl_verify", true)
+                .set("ssl_verify_hostname", false)
+                .set("ssl_trusted_ca_cert_data", FTP_TEST_SSL_TRUSTED_CA_CERT_DATA)
+                .set("path_prefix", "my-prefix")
+                .set("file_ext", ".csv")
+                .set("formatter", formatterConfig());
+
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+
+        runner.transaction(config, schema, 0, new Control());
+    }
+
     @Test
     public void testResume()
     {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-output-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Satoshi Akama
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-29 00:00:00.000000000 Z
+date: 2016-08-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -63,13 +63,11 @@ files:
 - src/main/java/org/embulk/output/ftp/SSLPlugins.java
 - src/main/java/org/embulk/output/ftp/TrustManagers.java
 - src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java
-- src/test/java/org/embulk/output/ftp/TestSSLPlugins.java
-- src/test/java/org/embulk/output/ftp/TrustedManagers.java
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
 - classpath/bcpkix-jdk15on-1.52.jar
 - classpath/bcprov-jdk15on-1.52.jar
-- classpath/embulk-output-ftp-0.1.4.jar
+- classpath/embulk-output-ftp-0.1.5.jar
 - classpath/ftp4j-1.7.2.jar
 homepage: https://github.com/embulk/embulk-output-ftp
 licenses:

data/src/test/java/org/embulk/output/ftp/TestSSLPlugins.java

@@ -1,5 +0,0 @@
-package org.embulk.output.ftp;
-
-public class TestSSLPlugins
-{
-}

data/src/test/java/org/embulk/output/ftp/TrustedManagers.java

@@ -1,5 +0,0 @@
-package org.embulk.output.ftp;
-
-public class TrustedManagers
-{
-}