embulk-input-ftp 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84b6cf71034980db6a9d4f78d5affdaae19aa792
-  data.tar.gz: 104382478ca9b24c2c08cc839637083a7540b715
+  metadata.gz: 4777eb105f91266e7be9329f4fe7c275e3aa4753
+  data.tar.gz: 1d797e96b71babe96627dd68291d7b8d2d046106
 SHA512:
-  metadata.gz: c98936c59b2c4ea47cb394462a6ab5df1a2d9fa1f4223d2c26ac9d75efde5746dac49267a668bd95657c5b19a797195a573a9984d7cdb6964725fc74cc1ade18
-  data.tar.gz: 035fbcd64345982d6370c3e38c607121e5b97a27daf26daf9274f4de441f5a10c4a61fa667120f6d0bbed0d2854a4983c3bb701428cd12d1fb832d4e7b84ddcc
+  metadata.gz: aa87efee4fb5dc309366b4394a958340941f704dbd03e1cbdf4c5a29419b70a87422163c8e82e5b7d4bc2b18d1a6cd6ad1eba62aa626aab9f8c12ef8703fe03a
+  data.tar.gz: 2794aec32f1da6644d305d585d1fda1e18a60325d1c3dc561b6b6e5767c471a21c8c0e215d363667afcdbfa325c4beebc8ea6bcd4366691422c1ed1a970777b5

data/ChangeLog

@@ -1,9 +1,15 @@
 
+Release 0.1.2 - 2015-06-10
+
+* Added support for SSL certificate verification
+
+
 Release 0.1.1 - 2015-04-29
 
 * Added support for SSL
 * Fixed last_path handling
 
+
 Release 0.1.0 - 2015-04-29
 
 * First release

data/README.md

@@ -16,10 +16,26 @@
 - **passive_mode**: use passive mode (boolean, default: true)
 - **ascii_mode**: use ASCII mode instead of binary mode (boolean, default: false)
 - **ssl**: use FTPS (SSL encryption). (boolean, default: false)
-  - Currently, it doesn't support certificate checking. Any certificate given by the remote host is trusted.
+- **ssl_verify**: verify the certification provided by the server. By default, connection fails if the server certification is not signed by one the CAs in JVM's default trusted CA list. (boolean, default: true)
+- **ssl_verify_hostname**: verify server's hostname matches with provided certificate. (boolean, default: true)
+- **ssl_trusted_ca_cert_file**: if the server certification is not signed by a certificate authority, set path to the X.508 certification file (pem file) of a private CA (string, optional)
+- **ssl_trusted_ca_cert_data**: similar to `ssl_trusted_ca_cert_file` but embed the contents of the PEM file as a string value instead of path to a local file (string, optional)
 
 ## Example
 
+Simple FTP:
+
+```yaml
+in:
+  type: ftp
+  host: ftp.example.net
+  port: 21
+  user: anonymous
+  path_prefix: /ftp/file/path/prefix
+```
+
+FTPS encryption without server certificate verification:
+
 ```yaml
 in:
   type: ftp
@@ -28,6 +44,37 @@ in:
   user: anonymous
   password: "mypassword"
   path_prefix: /ftp/file/path/prefix
+
+  ssl: true
+  ssl_verify: false
+```
+
+FTPS encryption with server certificate verification:
+
+```yaml
+in:
+  type: ftp
+  host: ftp.example.net
+  port: 21
+  user: anonymous
+  password: "mypassword"
+  path_prefix: /ftp/file/path/prefix
+
+  ssl: true
+  ssl_verify: true
+
+  ssl_verify_hostname: false   # to disable server hostname verification (optional)
+
+  # if the server use self-signed certificate, or set path to the pem file (optional)
+  ssl_trusted_ca_cert_file: /path/to/ca_cert.pem
+
+  # or embed contents of the pem file here (optional)
+  ssl_trusted_ca_cert_data: |
+      -----BEGIN CERTIFICATE-----
+      MIIFV...
+      ...
+      ...
+      -----END CERTIFICATE-----
 ```
 
 ## Build

data/build.gradle

@@ -12,12 +12,13 @@ configurations {
     provided
 }
 
-version = "0.1.1"
+version = "0.1.2"
 
 dependencies {
-    compile  "org.embulk:embulk-core:0.6.5"
-    provided "org.embulk:embulk-core:0.6.5"
+    compile  "org.embulk:embulk-core:0.6.8"
+    provided "org.embulk:embulk-core:0.6.8"
     compile files("libs/ftp4j-1.7.2.jar")
+    compile "org.bouncycastle:bcpkix-jdk15on:1.52"
     testCompile "junit:junit:4.+"
 }
 

data/src/main/java/org/embulk/input/FtpFileInputPlugin.java

@@ -9,14 +9,6 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
 import java.nio.channels.Channels;
-import java.security.SecureRandom;
-import java.security.KeyManagementException;
-import java.security.NoSuchAlgorithmException;
-import java.security.cert.X509Certificate;
-import javax.net.ssl.SSLContext;
-import javax.net.ssl.SSLSocketFactory;
-import javax.net.ssl.TrustManager;
-import javax.net.ssl.X509TrustManager;
 import org.slf4j.Logger;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.common.collect.ImmutableList;
@@ -46,11 +38,14 @@ import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.ResumableInputStream;
+import org.embulk.spi.util.RetryExecutor.Retryable;
+import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
 import org.embulk.input.ftp.BlockingTransfer;
-import org.embulk.input.ftp.RetryableInputStream;
-import org.embulk.input.ftp.RetryExecutor.Retryable;
-import org.embulk.input.ftp.RetryExecutor.RetryGiveupException;
-import static org.embulk.input.ftp.RetryExecutor.retryExecutor;
+import org.embulk.input.ftp.SSLPlugins;
+import org.embulk.input.ftp.SSLPlugins.SSLPluginTask;
+import org.embulk.input.ftp.SSLPlugins.SSLPluginConfig;
+import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public class FtpFileInputPlugin
         implements FileInputPlugin
@@ -58,7 +53,7 @@ public class FtpFileInputPlugin
     private final Logger log = Exec.getLogger(FtpFileInputPlugin.class);
 
     public interface PluginTask
-            extends Task
+            extends Task, SSLPlugins.SSLPluginTask
     {
         @Config("path_prefix")
         public String getPathPrefix();
@@ -97,6 +92,9 @@ public class FtpFileInputPlugin
         public List<String> getFiles();
         public void setFiles(List<String> files);
 
+        public SSLPluginConfig getSSLConfig();
+        public void setSSLConfig(SSLPluginConfig config);
+
         @ConfigInject
         public BufferAllocator getBufferAllocator();
     }
@@ -106,6 +104,8 @@ public class FtpFileInputPlugin
     {
         PluginTask task = config.loadConfig(PluginTask.class);
 
+        task.setSSLConfig(SSLPlugins.configure(task));
+
         // list files recursively
         List<String> files = listFiles(log, task);
         task.setFiles(files);
@@ -157,7 +157,7 @@ public class FtpFileInputPlugin
         FTPClient client = new FTPClient();
         try {
             if (task.getSsl()) {
-                client.setSSLSocketFactory(newSSLSocketFactory(task));
+                client.setSSLSocketFactory(SSLPlugins.newSSLSocketFactory(task.getSSLConfig(), task.getHost()));
                 client.setSecurity(FTPClient.SECURITY_FTPS);
             }
 
@@ -242,40 +242,6 @@ public class FtpFileInputPlugin
         }
     }
 
-    private static SSLSocketFactory newSSLSocketFactory(PluginTask task)
-    {
-        // TODO certificate check
-
-        TrustManager[] trustManager = new TrustManager[] {
-            new X509TrustManager() {
-                public X509Certificate[] getAcceptedIssuers()
-                {
-                    return null;
-                }
-
-                public void checkClientTrusted(X509Certificate[] certs, String authType)
-                {
-                }
-
-                public void checkServerTrusted(X509Certificate[] certs, String authType)
-                {
-                }
-            }
-        };
-
-        try {
-            SSLContext context = SSLContext.getInstance("TLS");
-            context.init(null, trustManager, new SecureRandom());
-            return context.getSocketFactory();
-
-        } catch (NoSuchAlgorithmException ex) {
-            throw new RuntimeException(ex);
-
-        } catch (KeyManagementException ex) {
-            throw new RuntimeException(ex);
-        }
-    }
-
     private List<String> listFiles(Logger log, PluginTask task)
     {
         FTPClient client = newFTPClient(log, task);
@@ -507,15 +473,15 @@ public class FtpFileInputPlugin
         return Channels.newInputStream(t.getReaderChannel());
     }
 
-    private static class FtpRetryableOpener
-            implements RetryableInputStream.Opener
+    private static class FtpInputStreamReopener
+            implements ResumableInputStream.Reopener
     {
         private final Logger log;
         private final FTPClient client;
         private final ExecutorService executor;
         private final String path;
 
-        public FtpRetryableOpener(Logger log, FTPClient client, ExecutorService executor, String path)
+        public FtpInputStreamReopener(Logger log, FTPClient client, ExecutorService executor, String path)
         {
             this.log = log;
             this.client = client;
@@ -524,7 +490,7 @@ public class FtpFileInputPlugin
         }
 
         @Override
-        public InputStream open(final long offset, final Exception exception) throws IOException
+        public InputStream reopen(final long offset, final Exception closedCause) throws IOException
         {
             try {
                 return retryExecutor()
@@ -535,7 +501,7 @@ public class FtpFileInputPlugin
                         @Override
                         public InputStream call() throws InterruptedIOException
                         {
-                            log.warn(String.format("FTP read failed. Retrying GET request with %,d bytes offset", offset), exception);
+                            log.warn(String.format("FTP read failed. Retrying GET request with %,d bytes offset", offset), closedCause);
                             return startDownload(log, client, path, offset, executor);
                         }
 
@@ -603,9 +569,9 @@ public class FtpFileInputPlugin
             }
             opened = true;
 
-            return new RetryableInputStream(
+            return new ResumableInputStream(
                     startDownload(log, client, path, 0L, executor),
-                    new FtpRetryableOpener(log, client, executor, path));
+                    new FtpInputStreamReopener(log, client, executor, path));
         }
 
         @Override

data/src/main/java/org/embulk/input/ftp/SSLPlugins.java

@@ -0,0 +1,245 @@
+package org.embulk.input.ftp;
+
+import java.util.List;
+import java.io.Reader;
+import java.io.FileReader;
+import java.io.StringReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.security.KeyStore;
+import java.security.NoSuchAlgorithmException;
+import java.security.GeneralSecurityException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateEncodingException;
+import java.security.KeyManagementException;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.X509TrustManager;
+import com.google.common.base.Optional;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.embulk.config.Config;
+import org.embulk.config.ConfigDefault;
+import org.embulk.config.ConfigException;
+
+public class SSLPlugins
+{
+    // SSLPlugins is only for SSL clients. SSL server implementation is out ouf scope.
+
+    public interface SSLPluginTask
+    {
+        @Config("ssl_verify")
+        @ConfigDefault("null")
+        public Optional<Boolean> getSslVerify();
+
+        @Config("ssl_verify_hostname")
+        @ConfigDefault("true")
+        public boolean getSslVerifyHostname();
+
+        @Config("ssl_trusted_ca_cert_file")
+        @ConfigDefault("null")
+        public Optional<String> getSslTrustedCaCertFile();
+
+        @Config("ssl_trusted_ca_cert_data")
+        @ConfigDefault("null")
+        public Optional<String> getSslTrustedCaCertData();
+    }
+
+    private static enum VerifyMode
+    {
+        NO_VERIFY,
+        CERTIFICATES,
+        JVM_DEFAULT;
+    }
+
+    public static class SSLPluginConfig
+    {
+        static SSLPluginConfig NO_VERIFY = new SSLPluginConfig(VerifyMode.NO_VERIFY, false, ImmutableList.<byte[]>of());
+
+        private final VerifyMode verifyMode;
+        private final boolean verifyHostname;
+        private final List<X509Certificate> certificates;
+
+        @JsonCreator
+        private SSLPluginConfig(
+            @JsonProperty("verifyMode") VerifyMode verifyMode,
+            @JsonProperty("verifyHostname") boolean verifyHostname,
+            @JsonProperty("certificates") List<byte[]> certificates)
+        {
+            this.verifyMode = verifyMode;
+            this.verifyHostname = verifyHostname;
+            this.certificates = ImmutableList.copyOf(
+                    Lists.transform(certificates, new Function<byte[], X509Certificate>() {
+                        public X509Certificate apply(byte[] data)
+                        {
+                            try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
+                                CertificateFactory cf = CertificateFactory.getInstance("X.509");
+                                return (X509Certificate) cf.generateCertificate(in);
+                            } catch (IOException | CertificateException ex) {
+                                throw new RuntimeException(ex);
+                            }
+                        }
+                    })
+                );
+        }
+
+        SSLPluginConfig(List<X509Certificate> certificates, boolean verifyHostname)
+        {
+            this.verifyMode = VerifyMode.CERTIFICATES;
+            this.verifyHostname = verifyHostname;
+            this.certificates = certificates;
+        }
+
+        static SSLPluginConfig useJvmDefault(boolean verifyHostname)
+        {
+            return new SSLPluginConfig(VerifyMode.JVM_DEFAULT, verifyHostname, ImmutableList.<byte[]>of());
+        }
+
+        @JsonProperty("verifyMode")
+        private VerifyMode getVerifyMode()
+        {
+            return verifyMode;
+        }
+
+        @JsonProperty("verifyHostname")
+        private boolean getVerifyHostname()
+        {
+            return verifyHostname;
+        }
+
+        @JsonProperty("certificates")
+        private List<byte[]> getCertData()
+        {
+            return Lists.transform(certificates, new Function<X509Certificate, byte[]>() {
+                public byte[] apply(X509Certificate cert)
+                {
+                    try {
+                        return cert.getEncoded();
+                    } catch (CertificateEncodingException ex) {
+                        throw new RuntimeException(ex);
+                    }
+                }
+            });
+        }
+
+        @JsonIgnore
+        public X509TrustManager[] newTrustManager()
+        {
+            try {
+                switch (verifyMode) {
+                case NO_VERIFY:
+                    return new X509TrustManager[] { getNoVerifyTrustManager() };
+                case CERTIFICATES:
+                    return TrustManagers.newTrustManager(certificates);
+                default: // JVM_DEFAULT
+                    return TrustManagers.newDefaultJavaTrustManager();
+                }
+            } catch (IOException | GeneralSecurityException ex) {
+                throw new RuntimeException(ex);
+            }
+        }
+    }
+
+    public static enum DefaultVerifyMode
+    {
+        VERIFY_BY_JVM_TRUSTED_CA_CERTS,
+        NO_VERIFY;
+    };
+
+    public static SSLPluginConfig configure(SSLPluginTask task)
+    {
+        return configure(task, DefaultVerifyMode.VERIFY_BY_JVM_TRUSTED_CA_CERTS);
+    }
+
+    public static SSLPluginConfig configure(SSLPluginTask task, DefaultVerifyMode defaultVerifyMode)
+    {
+        boolean verify = task.getSslVerify().or(defaultVerifyMode != DefaultVerifyMode.NO_VERIFY);
+        if (verify) {
+            Optional<List<X509Certificate>> certs = readTrustedCertificates(task);
+            if (certs.isPresent()) {
+                return new SSLPluginConfig(certs.get(), task.getSslVerifyHostname());
+            } else {
+                return SSLPluginConfig.useJvmDefault(task.getSslVerifyHostname());
+            }
+        } else {
+            return SSLPluginConfig.NO_VERIFY;
+        }
+    }
+
+    private static Optional<List<X509Certificate>> readTrustedCertificates(SSLPluginTask task)
+    {
+        String optionName;
+        Reader reader;
+        if (task.getSslTrustedCaCertData().isPresent()) {
+            optionName = "ssl_trusted_ca_cert_data";
+            reader = new StringReader(task.getSslTrustedCaCertData().get());
+        } else if (task.getSslTrustedCaCertFile().isPresent()) {
+            optionName = "ssl_trusted_ca_cert_file '" + task.getSslTrustedCaCertFile().get() + "'";
+            try {
+                reader = new FileReader(task.getSslTrustedCaCertFile().get());
+            } catch (IOException ex) {
+                throw new ConfigException("Failed to open "+optionName, ex);
+            }
+        } else {
+            return Optional.absent();
+        }
+
+        List<X509Certificate> certs;
+        try (Reader r = reader) {
+            certs = TrustManagers.readPemEncodedX509Certificates(r);
+            if (certs.isEmpty()) {
+                throw new ConfigException(optionName + " does not include valid X.509 PEM certificates");
+            }
+        } catch (CertificateException | IOException ex) {
+            throw new ConfigException("Failed to read "+optionName, ex);
+        }
+
+        return Optional.of(certs);
+    }
+
+    public static SSLSocketFactory newSSLSocketFactory(SSLPluginConfig config, String hostname)
+    {
+        try {
+            return TrustManagers.newSSLSocketFactory(
+                    null,  // TODO sending client certificate is not implemented yet
+                    config.newTrustManager(),
+                    config.getVerifyHostname() ? hostname : null);
+        } catch (KeyManagementException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    private static class NoVerifyTrustManager
+            implements X509TrustManager
+    {
+        static final NoVerifyTrustManager INSTANCE = new NoVerifyTrustManager();
+
+        private NoVerifyTrustManager()
+        { }
+
+        @Override
+        public X509Certificate[] getAcceptedIssuers()
+        {
+            return null;
+        }
+
+        @Override
+        public void checkClientTrusted(X509Certificate[] certs, String authType)
+        { }
+
+        @Override
+        public void checkServerTrusted(X509Certificate[] certs, String authType)
+        { }
+    }
+
+    private static X509TrustManager getNoVerifyTrustManager()
+    {
+        return NoVerifyTrustManager.INSTANCE;
+    }
+}

data/src/main/java/org/embulk/input/ftp/TrustManagers.java

@@ -0,0 +1,276 @@
+package org.embulk.input.ftp;
+
+import java.util.List;
+import java.util.ArrayList;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.Reader;
+import java.io.IOException;
+import java.net.Socket;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.security.KeyStore;
+import java.security.SecureRandom;
+import java.security.KeyStoreException;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.InvalidAlgorithmParameterException;
+import java.security.cert.Certificate;
+import java.security.cert.TrustAnchor;
+import java.security.cert.PKIXParameters;
+import java.security.cert.X509Certificate;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateParsingException;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSession;
+import javax.net.ssl.SSLParameters;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.KeyManager;
+import javax.net.ssl.X509TrustManager;
+import javax.net.ssl.TrustManagerFactory;
+import javax.net.ssl.SSLSocket;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.HostnameVerifier;
+import org.bouncycastle.openssl.PEMParser;
+import org.bouncycastle.openssl.PEMException;
+import org.bouncycastle.cert.X509CertificateHolder;
+import org.bouncycastle.cert.jcajce.JcaX509CertificateConverter;
+import sun.security.ssl.SSLSocketImpl;
+
+public class TrustManagers
+{
+    public static KeyStore readDefaultJavaKeyStore()
+            throws IOException, KeyStoreException, CertificateException
+    {
+        String path = (System.getProperty("java.home") + "/lib/security/cacerts").replace('/', File.separatorChar);
+        try {
+            KeyStore keyStore = KeyStore.getInstance("JKS");
+            try (FileInputStream in = new FileInputStream(path)) {
+                keyStore.load(in, null);  // password=null because cacerts file is not encrypted
+            }
+            return keyStore;
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);  // TODO assertion exception?
+        }
+    }
+
+    public static List<X509Certificate> readDefaultJavaTrustedCertificates()
+            throws IOException, CertificateException, KeyStoreException, InvalidAlgorithmParameterException
+    {
+        KeyStore keyStore = readDefaultJavaKeyStore();
+        PKIXParameters params = new PKIXParameters(keyStore);
+        List<X509Certificate> certs = new ArrayList<>();
+        for (TrustAnchor trustAnchor : params.getTrustAnchors() ) {
+            certs.add(trustAnchor.getTrustedCert());
+        }
+        return certs;
+    }
+
+    public static List<X509Certificate> readPemEncodedX509Certificates(Reader reader)
+            throws IOException, CertificateException
+    {
+        // this method abuses CertificateParsingException because its javadoc says
+        // CertificateParsingException is only for DER-encoded formats.
+
+        JcaX509CertificateConverter conv = new JcaX509CertificateConverter();
+        List<X509Certificate> certs = new ArrayList<>();
+
+        try {
+            PEMParser pemParser = new PEMParser(reader);
+            // PEMParser#close is unnecessary because it just closes underlying reader
+
+            while (true) {
+                Object pem = pemParser.readObject();
+
+                if (pem == null) {
+                    break;
+                }
+
+                if (pem instanceof X509CertificateHolder) {
+                    X509Certificate cert = conv.getCertificate((X509CertificateHolder) pem);
+                    certs.add(cert);
+                }
+            }
+
+        } catch (PEMException ex) {
+            // throw when parsing PemObject to Object fails
+            throw new CertificateParsingException(ex);
+
+        } catch (IOException ex) {
+            if (ex.getClass().equals(IOException.class)) {
+                String message = ex.getMessage();
+                if (message.startsWith("unrecognised object: ")) {
+                    // thrown at org.bouncycastle.openssl.PemParser.readObject when key type (header of a pem) is
+                    // unknown.
+                    throw new CertificateParsingException(ex);
+                } else if (message.startsWith("-----END ") && message.endsWith(" not found")) {
+                    // thrown at org.bouncycastle.util.io.pem.PemReader.loadObject when a pem file format is invalid
+                    throw new CertificateParsingException(ex);
+                }
+            } else {
+                throw ex;
+            }
+        }
+
+        return certs;
+    }
+
+    public static KeyStore buildKeyStoreFromTrustedCertificates(List<X509Certificate> certificates)
+            throws KeyStoreException
+    {
+        KeyStore keyStore = KeyStore.getInstance("JKS");
+        try {
+            keyStore.load(null);
+        } catch (IOException | CertificateException | NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);
+        }
+        int i = 0;
+        for (X509Certificate cert : certificates) {
+            keyStore.setCertificateEntry("cert_" + i, cert);
+            i++;
+        }
+        return keyStore;
+    }
+
+    public static X509TrustManager[] newTrustManager(List<X509Certificate> trustedCertificates)
+            throws KeyStoreException
+    {
+        try {
+            TrustManagerFactory factory = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm());
+            KeyStore keyStore = buildKeyStoreFromTrustedCertificates(trustedCertificates);
+            factory.init(keyStore);
+            List<X509TrustManager> tms = new ArrayList<>();
+            for (TrustManager tm : factory.getTrustManagers()) {
+                if (tm instanceof X509TrustManager) {
+                    tms.add((X509TrustManager) tm);
+                }
+            }
+            return tms.toArray(new X509TrustManager[tms.size()]);
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);  // TODO assertion exception?
+        }
+    }
+
+    public static X509TrustManager[] newDefaultJavaTrustManager()
+            throws IOException, CertificateException, KeyStoreException, InvalidAlgorithmParameterException
+    {
+        return newTrustManager(readDefaultJavaTrustedCertificates());
+    }
+
+    public static SSLContext newSSLContext(KeyManager[] keyManager, X509TrustManager[] trustManager)
+            throws KeyManagementException
+    {
+        try {
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(
+                    keyManager,
+                    trustManager,
+                    new SecureRandom());
+            return context;
+
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
+    public static SSLSocketFactory newSSLSocketFactory(KeyManager[] keyManager, X509TrustManager[] trustManager, String verifyHostname)
+            throws KeyManagementException
+    {
+        SSLContext context = newSSLContext(keyManager, trustManager);
+        SSLSocketFactory factory = context.getSocketFactory();
+        if (verifyHostname == null) {
+            return factory;
+        } else {
+            return new VerifyHostNameSSLSocketFactory(factory, verifyHostname);
+        }
+    }
+
+    private static class VerifyHostNameSSLSocketFactory
+            extends SSLSocketFactory
+    {
+        private final SSLSocketFactory next;
+        private final String hostname;
+
+        public VerifyHostNameSSLSocketFactory(SSLSocketFactory next, String hostname)
+        {
+            this.next = next;
+            this.hostname = hostname;
+        }
+
+        @Override
+        public String[] getDefaultCipherSuites()
+        {
+            return next.getDefaultCipherSuites();
+        }
+
+        @Override
+        public String[] getSupportedCipherSuites()
+        {
+            return next.getSupportedCipherSuites();
+        }
+
+        @Override
+        public Socket createSocket(Socket s, String host, int port, boolean autoClose)
+                throws IOException
+        {
+            Socket sock = next.createSocket(s, host, port, autoClose);
+            setSSLParameters(sock, false);
+            return sock;
+        }
+
+        @Override
+        public Socket createSocket(String host, int port)
+                throws IOException, UnknownHostException
+        {
+            Socket sock = next.createSocket(host, port);
+            setSSLParameters(sock, false);
+            return sock;
+        }
+
+        @Override
+        public Socket createSocket(String host, int port, InetAddress localHost, int localPort)
+                throws IOException, UnknownHostException
+        {
+            Socket sock = next.createSocket(host, port, localHost, localPort);
+            setSSLParameters(sock, false);
+            return sock;
+        }
+
+        @Override
+        public Socket createSocket(InetAddress host, int port)
+                throws IOException
+        {
+            Socket sock = next.createSocket(host, port);
+            setSSLParameters(sock, true);
+            return sock;
+        }
+
+        @Override
+        public Socket createSocket(InetAddress address, int port, InetAddress localAddress, int localPort)
+                throws IOException
+        {
+            Socket sock = next.createSocket(address, port, localAddress, localPort);
+            setSSLParameters(sock, true);
+            return sock;
+        }
+
+        private void setSSLParameters(Socket sock, boolean setHostname)
+        {
+            if (sock instanceof SSLSocket) {
+                SSLSocket s = (SSLSocket) sock;
+                String identAlgorithm = s.getSSLParameters().getEndpointIdentificationAlgorithm();
+                if (identAlgorithm != null && identAlgorithm.equalsIgnoreCase("HTTPS")) {
+                    // hostname verification is already configured.
+                } else {
+                    if (setHostname && s instanceof SSLSocketImpl) {
+                        ((SSLSocketImpl) s).setHost(hostname);
+                    }
+                    SSLParameters params = s.getSSLParameters();
+                    params.setEndpointIdentificationAlgorithm("HTTPS");
+                    s.setSSLParameters(params);
+                    // s.startHandshake
+                }
+            }
+        }
+    }
+}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-04-29 00:00:00.000000000 Z
+date: 2015-06-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -57,10 +57,12 @@ files:
 - libs/ftp4j-1.7.2.jar
 - src/main/java/org/embulk/input/FtpFileInputPlugin.java
 - src/main/java/org/embulk/input/ftp/BlockingTransfer.java
-- src/main/java/org/embulk/input/ftp/RetryExecutor.java
-- src/main/java/org/embulk/input/ftp/RetryableInputStream.java
+- src/main/java/org/embulk/input/ftp/SSLPlugins.java
+- src/main/java/org/embulk/input/ftp/TrustManagers.java
 - src/test/java/org/embulk/input/TestFtpFileInputPlugin.java
-- classpath/embulk-input-ftp-0.1.1.jar
+- classpath/bcpkix-jdk15on-1.52.jar
+- classpath/bcprov-jdk15on-1.52.jar
+- classpath/embulk-input-ftp-0.1.2.jar
 - classpath/ftp4j-1.7.2.jar
 homepage: https://github.com/embulk/embulk-input-ftp
 licenses:

data/src/main/java/org/embulk/input/ftp/RetryExecutor.java

@@ -1,131 +0,0 @@
-// TODO copied from S3FileInputPlugin. This should be moved to org.embulk.
-package org.embulk.input.ftp;
-
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-
-public class RetryExecutor
-{
-    public static RetryExecutor retryExecutor()
-    {
-        // TODO default configuration
-        return new RetryExecutor(3, 500, 30*60*1000);
-    }
-
-    public static class RetryGiveupException
-            extends ExecutionException
-    {
-        public RetryGiveupException(String message, Exception cause)
-        {
-            super(cause);
-        }
-
-        public RetryGiveupException(Exception cause)
-        {
-            super(cause);
-        }
-
-        public Exception getCause()
-        {
-            return (Exception) super.getCause();
-        }
-    }
-
-    public static interface Retryable<T>
-            extends Callable<T>
-    {
-        public T call()
-            throws Exception;
-
-        public boolean isRetryableException(Exception exception);
-
-        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
-            throws RetryGiveupException;
-
-        public void onGiveup(Exception firstException, Exception lastException)
-            throws RetryGiveupException;
-    }
-
-    private final int retryLimit;
-    private final int initialRetryWait;
-    private final int maxRetryWait;
-
-    private RetryExecutor(int retryLimit, int initialRetryWait, int maxRetryWait)
-    {
-        this.retryLimit = retryLimit;
-        this.initialRetryWait = initialRetryWait;
-        this.maxRetryWait = maxRetryWait;
-    }
-
-    public RetryExecutor withRetryLimit(int count)
-    {
-        return new RetryExecutor(count, initialRetryWait, maxRetryWait);
-    }
-
-    public RetryExecutor withInitialRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, msec, maxRetryWait);
-    }
-
-    public RetryExecutor withMaxRetryWait(int msec)
-    {
-        return new RetryExecutor(retryLimit, initialRetryWait, msec);
-    }
-
-    public <T> T runInterruptible(Retryable<T> op)
-            throws InterruptedException, RetryGiveupException
-    {
-        return run(op, true);
-    }
-
-    public <T> T run(Retryable<T> op)
-            throws RetryGiveupException
-    {
-        try {
-            return run(op, false);
-        } catch (InterruptedException ex) {
-            throw new RetryGiveupException("Unexpected interruption", ex);
-        }
-    }
-
-    private <T> T run(Retryable<T> op, boolean interruptible)
-            throws InterruptedException, RetryGiveupException
-    {
-        int retryWait = initialRetryWait;
-        int retryCount = 0;
-
-        Exception firstException = null;
-
-        while(true) {
-            try {
-                return op.call();
-            } catch (Exception exception) {
-                if (firstException == null) {
-                    firstException = exception;
-                }
-                if (!op.isRetryableException(exception) || retryCount >= retryLimit) {
-                    op.onGiveup(firstException, exception);
-                    throw new RetryGiveupException(firstException);
-                }
-
-                retryCount++;
-                op.onRetry(exception, retryCount, retryLimit, retryWait);
-
-                try {
-                    Thread.sleep(retryWait);
-                } catch (InterruptedException ex) {
-                    if (interruptible) {
-                        throw ex;
-                    }
-                }
-
-                // exponential back-off with hard limit
-                retryWait *= 2;
-                if (retryWait > maxRetryWait) {
-                    retryWait = maxRetryWait;
-                }
-            }
-        }
-    }
-}
-

data/src/main/java/org/embulk/input/ftp/RetryableInputStream.java

@@ -1,129 +0,0 @@
-// TODO copied from S3FileInputPlugin. This should be moved to org.embulk.
-package org.embulk.input.ftp;
-
-import java.io.InputStream;
-import java.io.IOException;
-
-public class RetryableInputStream
-        extends InputStream
-{
-    public interface Opener
-    {
-        public InputStream open(long offset, Exception exception) throws IOException;
-    }
-
-    private final Opener opener;
-    protected InputStream in;
-    private long offset;
-    private long markedOffset;
-
-    public RetryableInputStream(InputStream initialInputStream, Opener reopener)
-    {
-        this.opener = reopener;
-        this.in = initialInputStream;
-        this.offset = 0L;
-        this.markedOffset = 0L;
-    }
-
-    public RetryableInputStream(Opener opener) throws IOException
-    {
-        this(opener.open(0, null), opener);
-    }
-
-    private void reopen(Exception exception) throws IOException
-    {
-        if (in != null) {
-            in.close();
-            in = null;
-        }
-        in = opener.open(offset, exception);
-    }
-
-    @Override
-    public int read() throws IOException
-    {
-        while (true) {
-            try {
-                int v = in.read();
-                offset += 1;
-                return v;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int read(byte[] b) throws IOException
-    {
-        while (true) {
-            try {
-                int r = in.read(b);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int read(byte[] b, int off, int len) throws IOException
-    {
-        while (true) {
-            try {
-                int r = in.read(b, off, len);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public long skip(long n) throws IOException
-    {
-        while (true) {
-            try {
-                long r = in.skip(n);
-                offset += r;
-                return r;
-            } catch (IOException | RuntimeException ex) {
-                reopen(ex);
-            }
-        }
-    }
-
-    @Override
-    public int available() throws IOException
-    {
-        return in.available();
-    }
-
-    @Override
-    public void close() throws IOException
-    {
-        in.close();
-    }
-
-    @Override
-    public void mark(int readlimit)
-    {
-        in.mark(readlimit);
-        markedOffset = offset;
-    }
-
-    @Override
-    public void reset() throws IOException
-    {
-        in.reset();
-        offset = markedOffset;
-    }
-
-    @Override
-    public boolean markSupported()
-    {
-        return in.markSupported();
-    }
-}