embulk-input-ftp 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0333d873277223fc5c1d369fa1417e2481df6ac0
-  data.tar.gz: c3b61c7abde8c581303b993613e788378d133372
+  metadata.gz: 84b6cf71034980db6a9d4f78d5affdaae19aa792
+  data.tar.gz: 104382478ca9b24c2c08cc839637083a7540b715
 SHA512:
-  metadata.gz: 790052dc1eafaaa5aa093fa6ba0d12fe7ccc64b21de203fcbdc8495807b420549176e19524096a4b02765e26470cc7a1d04358c47e3878f59a6288fedb4941c6
-  data.tar.gz: 5e1755a9388bbeefb9c6a4e6f2c9e96833568a7ab6861fde7985b1c88f6995bd439a65e059f2c73175c102e845a61e35391f20e7f239abe40051aa646ecf405e
+  metadata.gz: c98936c59b2c4ea47cb394462a6ab5df1a2d9fa1f4223d2c26ac9d75efde5746dac49267a668bd95657c5b19a797195a573a9984d7cdb6964725fc74cc1ade18
+  data.tar.gz: 035fbcd64345982d6370c3e38c607121e5b97a27daf26daf9274f4de441f5a10c4a61fa667120f6d0bbed0d2854a4983c3bb701428cd12d1fb832d4e7b84ddcc

data/ChangeLog

@@ -1,4 +1,9 @@
 
+Release 0.1.1 - 2015-04-29
+
+* Added support for SSL
+* Fixed last_path handling
+
 Release 0.1.0 - 2015-04-29
 
 * First release

data/README.md

@@ -9,12 +9,14 @@
 ## Configuration
 
 - **host**: FTP server address (string, required)
-- **port**: FTP server port number (integer, default: 21)
+- **port**: FTP server port number (integer, default: `21`. `990` if `ssl` is true)
 - **user**: user name to login (string, optional)
 - **password**: password to login (string, default: `""`)
-- **path_prefix** prefix of target keys (string, required)
+- **path_prefix** prefix of target files (string, required)
 - **passive_mode**: use passive mode (boolean, default: true)
 - **ascii_mode**: use ASCII mode instead of binary mode (boolean, default: false)
+- **ssl**: use FTPS (SSL encryption). (boolean, default: false)
+  - Currently, it doesn't support certificate checking. Any certificate given by the remote host is trusted.
 
 ## Example
 

data/build.gradle

@@ -12,7 +12,7 @@ configurations {
     provided
 }
 
-version = "0.1.0"
+version = "0.1.1"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.6.5"
@@ -53,8 +53,8 @@ Gem::Specification.new do |spec|
   spec.name          = "${project.name}"
   spec.version       = "${project.version}"
   spec.authors       = ["Sadayuki Furuhashi"]
-  spec.summary       = %[Ftp file input plugin for Embulk]
-  spec.description   = %[Reads files stored on Ftp.]
+  spec.summary       = %[FTP file input plugin for Embulk]
+  spec.description   = %[Reads files stored on a FTP server.]
   spec.email         = ["frsyuki@gmail.com"]
   spec.licenses      = ["Apache 2.0"]
   spec.homepage      = "https://github.com/embulk/embulk-input-ftp"

data/src/main/java/org/embulk/input/FtpFileInputPlugin.java

@@ -9,6 +9,14 @@ import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.InputStream;
 import java.nio.channels.Channels;
+import java.security.SecureRandom;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.cert.X509Certificate;
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLSocketFactory;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import org.slf4j.Logger;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.google.common.collect.ImmutableList;
@@ -63,8 +71,8 @@ public class FtpFileInputPlugin
         public String getHost();
 
         @Config("port")
-        @ConfigDefault("21")
-        public int getPort();
+        @ConfigDefault("null")
+        public Optional<Integer> getPort();
 
         @Config("user")
         @ConfigDefault("null")
@@ -82,6 +90,10 @@ public class FtpFileInputPlugin
         @ConfigDefault("false")
         public boolean getAsciiMode();
 
+        @Config("ssl")
+        @ConfigDefault("false")
+        public boolean getSsl();
+
         public List<String> getFiles();
         public void setFiles(List<String> files);
 
@@ -144,7 +156,10 @@ public class FtpFileInputPlugin
     {
         FTPClient client = new FTPClient();
         try {
-            // TODO SSL
+            if (task.getSsl()) {
+                client.setSSLSocketFactory(newSSLSocketFactory(task));
+                client.setSecurity(FTPClient.SECURITY_FTPS);
+            }
 
             client.addCommunicationListener(new LoggingCommunicationListner(log));
 
@@ -164,7 +179,9 @@ public class FtpFileInputPlugin
             //client.setAutodetectUTF8
 
             log.info("Connecting to "+task.getHost());
-            client.connect(task.getHost(), task.getPort());
+            if (task.getPort().isPresent()) {
+                client.connect(task.getHost(), task.getPort().get());
+            }
 
             if (task.getUser().isPresent()) {
                 log.info("Logging in with user "+task.getUser().get());
@@ -225,6 +242,40 @@ public class FtpFileInputPlugin
         }
     }
 
+    private static SSLSocketFactory newSSLSocketFactory(PluginTask task)
+    {
+        // TODO certificate check
+
+        TrustManager[] trustManager = new TrustManager[] {
+            new X509TrustManager() {
+                public X509Certificate[] getAcceptedIssuers()
+                {
+                    return null;
+                }
+
+                public void checkClientTrusted(X509Certificate[] certs, String authType)
+                {
+                }
+
+                public void checkServerTrusted(X509Certificate[] certs, String authType)
+                {
+                }
+            }
+        };
+
+        try {
+            SSLContext context = SSLContext.getInstance("TLS");
+            context.init(null, trustManager, new SecureRandom());
+            return context.getSocketFactory();
+
+        } catch (NoSuchAlgorithmException ex) {
+            throw new RuntimeException(ex);
+
+        } catch (KeyManagementException ex) {
+            throw new RuntimeException(ex);
+        }
+    }
+
     private List<String> listFiles(Logger log, PluginTask task)
     {
         FTPClient client = newFTPClient(log, task);
@@ -267,7 +318,7 @@ public class FtpFileInputPlugin
 
             for (FTPFile file : client.list()) {
                 if (file.getName().startsWith(fileNamePrefix)) {
-                    listFilesRecursive(client, currentDirectory, file, builder);
+                    listFilesRecursive(client, currentDirectory, file, lastPath, builder);
                 }
             }
 
@@ -300,7 +351,7 @@ public class FtpFileInputPlugin
     }
 
     private static void listFilesRecursive(FTPClient client,
-            String baseDirectoryPath, FTPFile file,
+            String baseDirectoryPath, FTPFile file, Optional<String> lastPath,
             ImmutableList.Builder<String> builder)
         throws IOException, FTPException, FTPIllegalReplyException, FTPDataTransferException, FTPAbortedException, FTPListParseException
     {
@@ -309,6 +360,10 @@ public class FtpFileInputPlugin
         }
         String path = baseDirectoryPath + file.getName();
 
+        if (lastPath.isPresent() && path.compareTo(lastPath.get()) <= 0) {
+            return;
+        }
+
         switch (file.getType()) {
         case FTPFile.TYPE_FILE:
             builder.add(path);
@@ -316,7 +371,7 @@ public class FtpFileInputPlugin
         case FTPFile.TYPE_DIRECTORY:
             client.changeDirectory(path);
             for (FTPFile subFile : client.list()) {
-                listFilesRecursive(client, path, subFile, builder);
+                listFilesRecursive(client, path, subFile, lastPath, builder);
             }
             client.changeDirectory(baseDirectoryPath);
             break;
@@ -534,7 +589,7 @@ public class FtpFileInputPlugin
             this.client = newFTPClient(log, task);
             this.executor = Executors.newCachedThreadPool(
                     new ThreadFactoryBuilder()
-                        .setNameFormat("embulk-input-ftp-%d")
+                        .setNameFormat("embulk-input-ftp-transfer-%d")
                         .setDaemon(true)
                         .build());
             this.path = task.getFiles().get(taskIndex);

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -38,7 +38,7 @@ dependencies:
         version: '10.0'
   prerelease: false
   type: :development
-description: Reads files stored on Ftp.
+description: Reads files stored on a FTP server.
 email:
 - frsyuki@gmail.com
 executables: []
@@ -60,7 +60,7 @@ files:
 - src/main/java/org/embulk/input/ftp/RetryExecutor.java
 - src/main/java/org/embulk/input/ftp/RetryableInputStream.java
 - src/test/java/org/embulk/input/TestFtpFileInputPlugin.java
-- classpath/embulk-input-ftp-0.1.0.jar
+- classpath/embulk-input-ftp-0.1.1.jar
 - classpath/ftp4j-1.7.2.jar
 homepage: https://github.com/embulk/embulk-input-ftp
 licenses:
@@ -85,5 +85,5 @@ rubyforge_project:
 rubygems_version: 2.1.9
 signing_key:
 specification_version: 4
-summary: Ftp file input plugin for Embulk
+summary: FTP file input plugin for Embulk
 test_files: []