embulk-input-ftp 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0acbad097efd3f667a4ca8dafedb204826b513f2
-  data.tar.gz: b225f6ee1a38a587566717bdfc59ab24198afb43
+  metadata.gz: 14c9dbe5aaed1c2ff8e3e48ac58aa331a6b59370
+  data.tar.gz: 390c47334bce6509c343229f1ac43625b05923e3
 SHA512:
-  metadata.gz: 3b7e32fdaeb95bafb8205338e5d40e37325ca4a8846392670260a96f860fef33a2b879ef52d83722ec7a03991a05b78e5adc2b0e9b3bc3f0dddf8b1fdb61116b
-  data.tar.gz: 621c453bce9718b49bd7970a31cfba66391fb3e85755a6ca921a3f02697b212518ed211c6cef085d155b69fbbd2ce2e445da9e8cf15f8fd68d691fc225d1974c
+  metadata.gz: 632782394767ea3b3fccb39ac1b2cfed6904b360c159a339a81e047edc348301584b07e5d40267a124e0b20b1259149c192aa758c15a6073906c97f7947b5021
+  data.tar.gz: e8e221d03695924300cea6ae91fe70ed77b82a7f0760f0ce90208cda887842e1e0aa853bf660ab7a021edef68e81395dcfd029a4785a29c100af68130712a2da

data/.travis.yml

@@ -0,0 +1,5 @@
+language: java
+jdk:
+- oraclejdk8
+script:
+- ./gradlew --info check jacocoTestReport

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.1.4 - 2016-11-16
+
+* Added support for FTPES(FTPS explicit) 
+* Added support for incremental option
 
 Release 0.1.3 - 2015-08-18
 

data/README.md

@@ -2,6 +2,11 @@
 
 ## Overview
 
+This plugin support **FTP**, **FTPES(FTPS explicit)**, **FTPS(FTPS implicit)** and doesn't support **SFTP**.
+
+If you want to use SFTP, please use [embulk-input-sftp](https://github.com/embulk/embulk-input-sftp).
+
+
 * **Plugin type**: file input
 * **Resume supported**: yes
 * **Cleanup supported**: yes
@@ -9,18 +14,34 @@
 ## Configuration
 
 - **host**: FTP server address (string, required)
-- **port**: FTP server port number (integer, default: `21`. `990` if `ssl` is true)
+- **port**: FTP server port number (integer, default: `21`. `990` if `ssl` is true and `ssl_explicit` is false)
 - **user**: user name to login (string, optional)
 - **password**: password to login (string, default: `""`)
 - **path_prefix** prefix of target files (string, required)
+- **incremental** enables incremental loading(boolean, optional. default: true. If incremental loading is enabled, config diff for the next execution will include last_path parameter so that next execution skips files before the path. Otherwise, last_path will not be included.
 - **passive_mode**: use passive mode (boolean, default: true)
 - **ascii_mode**: use ASCII mode instead of binary mode (boolean, default: false)
 - **ssl**: use FTPS (SSL encryption). (boolean, default: false)
+- **ssl_explicit** use FTPS(explicit) instead of FTPS(implicit). (boolean, default:true)
 - **ssl_verify**: verify the certification provided by the server. By default, connection fails if the server certification is not signed by one the CAs in JVM's default trusted CA list. (boolean, default: true)
 - **ssl_verify_hostname**: verify server's hostname matches with provided certificate. (boolean, default: true)
 - **ssl_trusted_ca_cert_file**: if the server certification is not signed by a certificate authority, set path to the X.508 certification file (pem file) of a private CA (string, optional)
 - **ssl_trusted_ca_cert_data**: similar to `ssl_trusted_ca_cert_file` but embed the contents of the PEM file as a string value instead of path to a local file (string, optional)
 
+### FTP / FTPS default port number
+
+FTP and FTPS server usually listens following port number(TCP) as default.
+
+Please be sure to configure firewall rules.
+
+|                         | FTP | FTPS(explicit) = FTPES | FTPS(implicit) = FTPS |
+|:------------------------|----:|-----------------------:|----------------------:|
+| Control channel port    |  21 |                     21 |             990 (\*1) |
+| Data channel port (\*2) |  20 |                     20 |                   989 |
+
+1. If you're using both of FTPS(implicit) and FTP, server may also listen 21/TCP for unecnrypted FTP.
+2. If you're using `passive mode`, data channel port can be taken between 1024 and 65535.
+
 ## Example
 
 Simple FTP:

data/build.gradle

@@ -2,6 +2,7 @@ plugins {
     id "com.jfrog.bintray" version "1.1"
     id "com.github.jruby-gradle.base" version "0.1.5"
     id "java"
+    id "jacoco"
 }
 import com.github.jrubygradle.JRubyExec
 repositories {
@@ -12,7 +13,7 @@ configurations {
     provided
 }
 
-version = "0.1.3"
+version = "0.1.4"
 
 dependencies {
     compile  "org.embulk:embulk-core:0.7.0"

data/src/main/java/org/embulk/input/FtpFileInputPlugin.java

@@ -51,6 +51,9 @@ public class FtpFileInputPlugin
         implements FileInputPlugin
 {
     private final Logger log = Exec.getLogger(FtpFileInputPlugin.class);
+    private static final int FTP_DEFULAT_PORT = 21;
+    private static final int FTPS_DEFAULT_PORT = 990;
+    private static final int FTPES_DEFAULT_PORT = 21;
 
     public interface PluginTask
             extends Task, SSLPlugins.SSLPluginTask
@@ -62,6 +65,10 @@ public class FtpFileInputPlugin
         @ConfigDefault("null")
         public Optional<String> getLastPath();
 
+        @Config("incremental")
+        @ConfigDefault("true")
+        public boolean getIncremental();
+
         @Config("host")
         public String getHost();
 
@@ -89,6 +96,10 @@ public class FtpFileInputPlugin
         @ConfigDefault("false")
         public boolean getSsl();
 
+        @Config("ssl_explicit")
+        @ConfigDefault("true")
+        public boolean getSslExplicit();
+
         public List<String> getFiles();
         public void setFiles(List<String> files);
 
@@ -130,15 +141,17 @@ public class FtpFileInputPlugin
         ConfigDiff configDiff = Exec.newConfigDiff();
 
         // last_path
-        if (task.getFiles().isEmpty()) {
-            // keep the last value
-            if (task.getLastPath().isPresent()) {
-                configDiff.set("last_path", task.getLastPath().get());
+        if (task.getIncremental()) {
+            if (task.getFiles().isEmpty()) {
+                // keep the last value
+                if (task.getLastPath().isPresent()) {
+                    configDiff.set("last_path", task.getLastPath().get());
+                }
+            } else {
+                List<String> files = new ArrayList<String>(task.getFiles());
+                Collections.sort(files);
+                configDiff.set("last_path", files.get(files.size() - 1));
             }
-        } else {
-            List<String> files = new ArrayList<String>(task.getFiles());
-            Collections.sort(files);
-            configDiff.set("last_path", files.get(files.size() - 1));
         }
 
         return configDiff;
@@ -156,10 +169,21 @@ public class FtpFileInputPlugin
     {
         FTPClient client = new FTPClient();
         try {
+            int defaultPort = FTP_DEFULAT_PORT;
             if (task.getSsl()) {
                 client.setSSLSocketFactory(SSLPlugins.newSSLSocketFactory(task.getSSLConfig(), task.getHost()));
-                client.setSecurity(FTPClient.SECURITY_FTPS);
+                if (task.getSslExplicit()) {
+                    client.setSecurity(FTPClient.SECURITY_FTPES);
+                    defaultPort = FTPES_DEFAULT_PORT;
+                    log.info("Using FTPES(FTPS/explicit) mode");
+                }
+                else {
+                    client.setSecurity(FTPClient.SECURITY_FTPS);
+                    defaultPort = FTPS_DEFAULT_PORT;
+                    log.info("Using FTPS(FTPS/implicit) mode");
+                }
             }
+            int port = task.getPort().isPresent()? task.getPort().get() : defaultPort;
 
             client.addCommunicationListener(new LoggingCommunicationListner(log));
 
@@ -178,10 +202,8 @@ public class FtpFileInputPlugin
             //client.setDataTimeout
             //client.setAutodetectUTF8
 
-            log.info("Connecting to "+task.getHost());
-            if (task.getPort().isPresent()) {
-                client.connect(task.getHost(), task.getPort().get());
-            }
+            client.connect(task.getHost(), port);
+            log.info("Connecting to {}:{}",task.getHost(),port);
 
             if (task.getUser().isPresent()) {
                 log.info("Logging in with user "+task.getUser().get());

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-19 00:00:00.000000000 Z
+date: 2016-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: bundler
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.0'
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
+  name: bundler
   prerelease: false
   type: :development
-- !ruby/object:Gem::Dependency
-  name: rake
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '1.0'
+- !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
       - !ruby/object:Gem::Version
         version: '10.0'
+  name: rake
   prerelease: false
   type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '10.0'
 description: Reads files stored on a FTP server.
 email:
 - frsyuki@gmail.com
@@ -46,6 +46,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - ChangeLog
 - README.md
 - build.gradle
@@ -62,7 +63,7 @@ files:
 - src/test/java/org/embulk/input/TestFtpFileInputPlugin.java
 - classpath/bcpkix-jdk15on-1.52.jar
 - classpath/bcprov-jdk15on-1.52.jar
-- classpath/embulk-input-ftp-0.1.3.jar
+- classpath/embulk-input-ftp-0.1.4.jar
 - classpath/ftp4j-1.7.2.jar
 homepage: https://github.com/embulk/embulk-input-ftp
 licenses: