embulk-input-ftp 0.1.6 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 00c4a9fb76f69fd4c8cc0364d0204286c8fc00c8
-  data.tar.gz: 3c1c4545a7b51ae7832708b93af859fd12f5195a
+  metadata.gz: 27845469df3d11bb23cf8ef4a815733383a34768
+  data.tar.gz: 391a4e30104f40bb79d34b1780863bbac9fc32f0
 SHA512:
-  metadata.gz: 7bb065f223aceb3519b07eaef2454f92dbd4832f795a5177fbb1a0fb5fcd28bb794fa59b28baead1c98b547e119737f0e3acec1dc2bf326731e590d333672c7c
-  data.tar.gz: 45dc1c461576924388e9ee025ce9cc0a86536d07cba5e0daa69df11c152e767bc43002e291a69be2baaecaab2acb64ed076fbedad75192a9ff8599ca87face6b
+  metadata.gz: 43298f5fba9022feb85f0523f99e703f69553fb47c1f1e41a0a34c78584bb162ff84291de2117367e26dae290d865bc90122e806baa8a10b1a3833ea2a5f3d2a
+  data.tar.gz: 663e55ceb928b929b00e8c53566617932b45dcdda8d4cb81d1c2b5a67ba2ed99387fbdc1d6fcd03b6e82e89c6615bacd27eb5d561b536f4be789659b1bc3cfe6

data/src/main/java/org/embulk/input/FtpFileInputPlugin.java

@@ -1,49 +1,52 @@
 package org.embulk.input;
 
-import java.util.List;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ExecutorService;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.io.InputStream;
-import java.nio.channels.Channels;
-import org.slf4j.Logger;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.common.collect.ImmutableList;
-import com.google.common.base.Optional;
-import com.google.common.base.Throwables;
 import com.google.common.base.Function;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import it.sauronsoftware.ftp4j.FTPAbortedException;
 import it.sauronsoftware.ftp4j.FTPClient;
-import it.sauronsoftware.ftp4j.FTPFile;
-import it.sauronsoftware.ftp4j.FTPConnector;
 import it.sauronsoftware.ftp4j.FTPCommunicationListener;
+import it.sauronsoftware.ftp4j.FTPConnector;
+import it.sauronsoftware.ftp4j.FTPDataTransferException;
 import it.sauronsoftware.ftp4j.FTPDataTransferListener;
 import it.sauronsoftware.ftp4j.FTPException;
+import it.sauronsoftware.ftp4j.FTPFile;
 import it.sauronsoftware.ftp4j.FTPIllegalReplyException;
-import it.sauronsoftware.ftp4j.FTPDataTransferException;
-import it.sauronsoftware.ftp4j.FTPAbortedException;
 import it.sauronsoftware.ftp4j.FTPListParseException;
-import org.embulk.config.TaskReport;
 import org.embulk.config.Config;
-import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigInject;
 import org.embulk.config.ConfigSource;
 import org.embulk.config.Task;
+import org.embulk.config.TaskReport;
 import org.embulk.config.TaskSource;
 import org.embulk.spi.BufferAllocator;
 import org.embulk.spi.Exec;
 import org.embulk.spi.FileInputPlugin;
 import org.embulk.spi.TransactionalFileInput;
 import org.embulk.spi.util.InputStreamFileInput;
+import org.embulk.spi.util.InputStreamFileInput.InputStreamWithHints;
 import org.embulk.spi.util.ResumableInputStream;
-import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
+import org.embulk.spi.util.RetryExecutor.Retryable;
 import org.embulk.util.ftp.BlockingTransfer;
 import org.embulk.util.ftp.SSLPlugins;
 import org.embulk.util.ftp.SSLPlugins.SSLPluginConfig;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InterruptedIOException;
+import java.nio.channels.Channels;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
 public class FtpFileInputPlugin
@@ -58,55 +61,55 @@ public class FtpFileInputPlugin
             extends Task, SSLPlugins.SSLPluginTask
     {
         @Config("path_prefix")
-        public String getPathPrefix();
+        String getPathPrefix();
 
         @Config("last_path")
         @ConfigDefault("null")
-        public Optional<String> getLastPath();
+        Optional<String> getLastPath();
 
         @Config("incremental")
         @ConfigDefault("true")
-        public boolean getIncremental();
+        boolean getIncremental();
 
         @Config("host")
-        public String getHost();
+        String getHost();
 
         @Config("port")
         @ConfigDefault("null")
-        public Optional<Integer> getPort();
+        Optional<Integer> getPort();
 
         @Config("user")
         @ConfigDefault("null")
-        public Optional<String> getUser();
+        Optional<String> getUser();
 
         @Config("password")
         @ConfigDefault("null")
-        public Optional<String> getPassword();
+        Optional<String> getPassword();
 
         @Config("passive_mode")
         @ConfigDefault("true")
-        public boolean getPassiveMode();
+        boolean getPassiveMode();
 
         @Config("ascii_mode")
         @ConfigDefault("false")
-        public boolean getAsciiMode();
+        boolean getAsciiMode();
 
         @Config("ssl")
         @ConfigDefault("false")
-        public boolean getSsl();
+        boolean getSsl();
 
         @Config("ssl_explicit")
         @ConfigDefault("true")
-        public boolean getSslExplicit();
+        boolean getSslExplicit();
 
-        public List<String> getFiles();
-        public void setFiles(List<String> files);
+        List<String> getFiles();
+        void setFiles(List<String> files);
 
-        public SSLPluginConfig getSSLConfig();
-        public void setSSLConfig(SSLPluginConfig config);
+        SSLPluginConfig getSSLConfig();
+        void setSSLConfig(SSLPluginConfig config);
 
         @ConfigInject
-        public BufferAllocator getBufferAllocator();
+        BufferAllocator getBufferAllocator();
     }
 
     @Override
@@ -146,7 +149,8 @@ public class FtpFileInputPlugin
                 if (task.getLastPath().isPresent()) {
                     configDiff.set("last_path", task.getLastPath().get());
                 }
-            } else {
+            }
+            else {
                 List<String> files = new ArrayList<String>(task.getFiles());
                 Collections.sort(files);
                 configDiff.set("last_path", files.get(files.size() - 1));
@@ -182,7 +186,7 @@ public class FtpFileInputPlugin
                     log.info("Using FTPS(FTPS/implicit) mode");
                 }
             }
-            int port = task.getPort().isPresent()? task.getPort().get() : defaultPort;
+            int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
 
             client.addCommunicationListener(new LoggingCommunicationListner(log));
 
@@ -202,11 +206,11 @@ public class FtpFileInputPlugin
             //client.setAutodetectUTF8
 
             client.connect(task.getHost(), port);
-            log.info("Connecting to {}:{}",task.getHost(),port);
+            log.info("Connecting to {}:{}", task.getHost(), port);
 
             if (task.getUser().isPresent()) {
-                log.info("Logging in with user "+task.getUser().get());
-                client.login(task.getUser().get(), task.getPassword().or(""));
+                log.info("Logging in with user " + task.getUser().get());
+                client.login(task.getUser().get(), task.getPassword().orElse(""));
             }
 
             log.info("Using passive mode");
@@ -215,7 +219,8 @@ public class FtpFileInputPlugin
             if (task.getAsciiMode()) {
                 log.info("Using ASCII mode");
                 client.setType(FTPClient.TYPE_TEXTUAL);
-            } else {
+            }
+            else {
                 log.info("Using binary mode");
                 client.setType(FTPClient.TYPE_BINARY);
             }
@@ -228,20 +233,20 @@ public class FtpFileInputPlugin
             FTPClient connected = client;
             client = null;
             return connected;
-
-        } catch (FTPException ex) {
-            log.info("FTP command failed: "+ex.getCode()+" "+ex.getMessage());
+        }
+        catch (FTPException ex) {
+            log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
             throw Throwables.propagate(ex);
-
-        } catch (FTPIllegalReplyException ex) {
+        }
+        catch (FTPIllegalReplyException ex) {
             log.info("FTP protocol error");
             throw Throwables.propagate(ex);
-
-        } catch (IOException ex) {
-            log.info("FTP network error: "+ex);
+        }
+        catch (IOException ex) {
+            log.info("FTP network error: " + ex);
             throw Throwables.propagate(ex);
-
-        } finally {
+        }
+        finally {
             if (client != null) {
                 disconnectClient(client);
             }
@@ -253,11 +258,14 @@ public class FtpFileInputPlugin
         if (client.isConnected()) {
             try {
                 client.disconnect(false);
-            } catch (FTPException ex) {
+            }
+            catch (FTPException ex) {
                 // do nothing
-            } catch (FTPIllegalReplyException ex) {
+            }
+            catch (FTPIllegalReplyException ex) {
                 // do nothing
-            } catch (IOException ex) {
+            }
+            catch (IOException ex) {
                 // do nothing
             }
         }
@@ -268,7 +276,8 @@ public class FtpFileInputPlugin
         FTPClient client = newFTPClient(log, task);
         try {
             return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath());
-        } finally {
+        }
+        finally {
             disconnectClient(client);
         }
     }
@@ -281,12 +290,14 @@ public class FtpFileInputPlugin
         if (prefix.isEmpty()) {
             directory = "";
             fileNamePrefix = "";
-        } else {
+        }
+        else {
             int pos = prefix.lastIndexOf("/");
             if (pos < 0) {
                 directory = "";
                 fileNamePrefix = prefix;
-            } else {
+            }
+            else {
                 directory = prefix.substring(0, pos + 1);  // include last "/"
                 fileNamePrefix = prefix.substring(pos + 1);
             }
@@ -308,29 +319,29 @@ public class FtpFileInputPlugin
                     listFilesRecursive(client, currentDirectory, file, lastPath, builder);
                 }
             }
-
-        } catch (FTPListParseException ex) {
+        }
+        catch (FTPListParseException ex) {
             log.info("FTP listing files failed");
             throw Throwables.propagate(ex);
-
-        } catch (FTPAbortedException ex) {
+        }
+        catch (FTPAbortedException ex) {
             log.info("FTP listing files failed");
             throw Throwables.propagate(ex);
-
-        } catch (FTPDataTransferException ex) {
+        }
+        catch (FTPDataTransferException ex) {
             log.info("FTP data transfer failed");
             throw Throwables.propagate(ex);
-
-        } catch (FTPException ex) {
-            log.info("FTP command failed: "+ex.getCode()+" "+ex.getMessage());
+        }
+        catch (FTPException ex) {
+            log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
             throw Throwables.propagate(ex);
-
-        } catch (FTPIllegalReplyException ex) {
+        }
+        catch (FTPIllegalReplyException ex) {
             log.info("FTP protocol error");
             throw Throwables.propagate(ex);
-
-        } catch (IOException ex) {
-            log.info("FTP network error: "+ex);
+        }
+        catch (IOException ex) {
+            log.info("FTP network error: " + ex);
             throw Throwables.propagate(ex);
         }
 
@@ -386,7 +397,7 @@ public class FtpFileInputPlugin
 
         public void received(String statement)
         {
-            log.info("< "+statement);
+            log.info("< " + statement);
         }
 
         public void sent(String statement)
@@ -395,7 +406,7 @@ public class FtpFileInputPlugin
                 // don't show password
                 return;
             }
-            log.info("> "+statement);
+            log.info("> " + statement);
         }
     }
 
@@ -424,14 +435,14 @@ public class FtpFileInputPlugin
         {
             totalTransfer += length;
             if (totalTransfer > nextTransferNotice) {
-                log.info("Transferred "+totalTransfer+" bytes");
-                nextTransferNotice = ((totalTransfer / transferNoticeBytes)+1) * transferNoticeBytes;
+                log.info("Transferred " + totalTransfer + " bytes");
+                nextTransferNotice = ((totalTransfer / transferNoticeBytes) + 1) * transferNoticeBytes;
             }
         }
 
         public void completed()
         {
-            log.info("Transfer completed "+totalTransfer+" bytes");
+            log.info("Transfer completed " + totalTransfer + " bytes");
         }
 
         public void aborted()
@@ -445,7 +456,7 @@ public class FtpFileInputPlugin
         }
     }
 
-    private static final long TRANSFER_NOTICE_BYTES = 100*1024*1024;
+    private static final long TRANSFER_NOTICE_BYTES = 100 * 1024 * 1024;
 
     private static InputStream startDownload(final Logger log, final FTPClient client,
             final String path, final long offset, ExecutorService executor)
@@ -460,30 +471,31 @@ public class FtpFileInputPlugin
                             {
                                 try {
                                     client.download(path, Channels.newOutputStream(transfer.getWriterChannel()), offset, new LoggingTransferListener(log, TRANSFER_NOTICE_BYTES));
-
-                                } catch (FTPException ex) {
-                                    log.info("FTP command failed: "+ex.getCode()+" "+ex.getMessage());
+                                }
+                                catch (FTPException ex) {
+                                    log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
                                     throw Throwables.propagate(ex);
-
-                                } catch (FTPDataTransferException ex) {
+                                }
+                                catch (FTPDataTransferException ex) {
                                     log.info("FTP data transfer failed");
                                     throw Throwables.propagate(ex);
-
-                                } catch (FTPAbortedException ex) {
+                                }
+                                catch (FTPAbortedException ex) {
                                     log.info("FTP listing files failed");
                                     throw Throwables.propagate(ex);
-
-                                } catch (FTPIllegalReplyException ex) {
+                                }
+                                catch (FTPIllegalReplyException ex) {
                                     log.info("FTP protocol error");
                                     throw Throwables.propagate(ex);
-
-                                } catch (IOException ex) {
+                                }
+                                catch (IOException ex) {
                                     throw Throwables.propagate(ex);
-
-                                } finally {
+                                }
+                                finally {
                                     try {
                                         transfer.getWriterChannel().close();
-                                    } catch (IOException ex) {
+                                    }
+                                    catch (IOException ex) {
                                         throw new RuntimeException(ex);
                                     }
                                 }
@@ -517,7 +529,7 @@ public class FtpFileInputPlugin
                 return retryExecutor()
                     .withRetryLimit(3)
                     .withInitialRetryWait(500)
-                    .withMaxRetryWait(30*1000)
+                    .withMaxRetryWait(30 * 1000)
                     .runInterruptible(new Retryable<InputStream>() {
                         @Override
                         public InputStream call() throws InterruptedIOException
@@ -537,10 +549,11 @@ public class FtpFileInputPlugin
                                 throws RetryGiveupException
                         {
                             String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
-                                    retryCount, retryLimit, retryWait/1000, exception.getMessage());
+                                    retryCount, retryLimit, retryWait / 1000, exception.getMessage());
                             if (retryCount % 3 == 0) {
                                 log.warn(message, exception);
-                            } else {
+                            }
+                            else {
                                 log.warn(message);
                             }
                         }
@@ -551,10 +564,12 @@ public class FtpFileInputPlugin
                         {
                         }
                     });
-            } catch (RetryGiveupException ex) {
+            }
+            catch (RetryGiveupException ex) {
                 Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
                 throw Throwables.propagate(ex.getCause());
-            } catch (InterruptedException ex) {
+            }
+            catch (InterruptedException ex) {
                 throw new InterruptedIOException();
             }
         }
@@ -583,16 +598,18 @@ public class FtpFileInputPlugin
         }
 
         @Override
-        public InputStream openNext() throws IOException
+        public InputStreamWithHints openNextWithHints() throws IOException
         {
             if (opened) {
                 return null;
             }
             opened = true;
 
-            return new ResumableInputStream(
-                    startDownload(log, client, path, 0L, executor),
-                    new FtpInputStreamReopener(log, client, executor, path));
+            return new InputStreamWithHints(
+                    new ResumableInputStream(
+                            startDownload(log, client, path, 0L, executor),
+                            new FtpInputStreamReopener(log, client, executor, path)), path
+            );
         }
 
         @Override
@@ -600,7 +617,8 @@ public class FtpFileInputPlugin
         {
             try {
                 executor.shutdownNow();
-            } finally {
+            }
+            finally {
                 disconnectClient(client);
             }
         }
@@ -615,7 +633,9 @@ public class FtpFileInputPlugin
             super(task.getBufferAllocator(), new SingleFileProvider(log, task, taskIndex));
         }
 
-        public void abort() { }
+        public void abort()
+        {
+        }
 
         public TaskReport commit()
         {

data/src/test/java/org/embulk/input/TestFtpFileInputPlugin.java

@@ -1,5 +1,313 @@
 package org.embulk.input;
 
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.embulk.EmbulkTestRuntime;
+import org.embulk.config.ConfigDiff;
+import org.embulk.config.ConfigSource;
+import org.embulk.config.TaskReport;
+import org.embulk.config.TaskSource;
+import org.embulk.input.FtpFileInputPlugin.PluginTask;
+import org.embulk.spi.Exec;
+import org.embulk.spi.FileInputPlugin;
+import org.embulk.spi.FileInputRunner;
+import org.embulk.spi.InputPlugin;
+import org.embulk.spi.Schema;
+import org.embulk.spi.TestPageBuilderReader;
+import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
+import org.embulk.spi.util.Pages;
+import org.embulk.standards.CsvParserPlugin;
+import org.embulk.util.ftp.SSLPlugins;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.slf4j.Logger;
+
+import java.lang.reflect.Method;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.junit.Assert.assertThat;
+
 public class TestFtpFileInputPlugin
 {
+    private static String FTP_TEST_HOST;
+    private static Integer FTP_TEST_PORT;
+    private static Integer FTP_TEST_SSL_PORT;
+    private static String FTP_TEST_USER;
+    private static String FTP_TEST_PASSWORD;
+    private static String FTP_TEST_SSL_TRUSTED_CA_CERT_FILE;
+    private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
+    private static String FTP_TEST_DIRECTORY;
+    private static String FTP_TEST_PATH_PREFIX;
+    private FileInputRunner runner;
+    private TestPageBuilderReader.MockPageOutput output;
+
+    /*
+     * This test case requires environment variables
+     *   FTP_TEST_HOST
+     *   FTP_TEST_USER
+     *   FTP_TEST_PASSWORD
+     *   FTP_TEST_SSL_TRUSTED_CA_CERT_FILE
+     */
+    @BeforeClass
+    public static void initializeConstant()
+    {
+        final Map<String, String> env = System.getenv();
+        FTP_TEST_HOST = env.getOrDefault("FTP_TEST_HOST", "localhost");
+        FTP_TEST_PORT = Integer.valueOf(env.getOrDefault("FTP_TEST_PORT", "11021"));
+        FTP_TEST_SSL_PORT = Integer.valueOf(env.getOrDefault("FTP_TEST_SSL_PORT", "990"));
+        FTP_TEST_USER = env.getOrDefault("FTP_TEST_USER", "scott");
+        FTP_TEST_PASSWORD = env.getOrDefault("FTP_TEST_PASSWORD", "tiger");
+        FTP_TEST_SSL_TRUSTED_CA_CERT_FILE = env.getOrDefault("FTP_TEST_SSL_TRUSTED_CA_CERT_FILE", "dummy");
+        FTP_TEST_SSL_TRUSTED_CA_CERT_DATA = env.getOrDefault("FTP_TEST_SSL_TRUSTED_CA_CERT_DATA", "dummy");
+
+        FTP_TEST_DIRECTORY = getDirectory(env.getOrDefault("FTP_TEST_DIRECTORY", "/unittest/"));
+        FTP_TEST_PATH_PREFIX = FTP_TEST_DIRECTORY + "sample_";
+    }
+
+    @Rule
+    public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
+    private FtpFileInputPlugin plugin;
+
+    @Before
+    public void createResources()
+    {
+        plugin = new FtpFileInputPlugin();
+        runner = new FileInputRunner(runtime.getInstance(FtpFileInputPlugin.class));
+        output = new MockPageOutput();
+    }
+
+    @Test(expected = RuntimeException.class) // TODO ConfigException should be thrown
+    public void testTransactionWithInvalidHost()
+    {
+        ConfigSource config = config().deepCopy()
+                .set("host", "non-exists.example.com");
+
+        runner.transaction(config, new Control());
+    }
+
+    @Test
+    public void testResume()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        task.setSSLConfig(sslConfig(task));
+        task.setFiles(Arrays.asList("in/aa/a"));
+        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
+        {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                return emptyTaskReports(taskCount);
+            }
+        });
+        assertThat(configDiff.get(String.class, "last_path"), is("in/aa/a"));
+    }
+
+    @Test
+    public void testCleanup()
+    {
+        PluginTask task = config().loadConfig(PluginTask.class);
+        plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFilesWithNonExistPath() throws Exception
+    {
+        ConfigSource config = config().deepCopy()
+                .set("path_prefix", "non-exists-path");
+        PluginTask task = config.loadConfig(PluginTask.class);
+        plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                assertThat(taskCount, is(0));
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        method.setAccessible(true);
+        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
+        assertThat(fileList.size(), is(0));
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFiles() throws Exception
+    {
+        List<String> expected = Arrays.asList(
+            FTP_TEST_PATH_PREFIX + "01.csv",
+            FTP_TEST_PATH_PREFIX + "02.csv"
+        );
+
+        ConfigSource config = config();
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            {
+                assertThat(taskCount, is(2));
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        method.setAccessible(true);
+        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
+        assertThat(fileList.get(0), is(expected.get(0)));
+        assertThat(fileList.get(1), is(expected.get(1)));
+        assertThat(configDiff.get(String.class, "last_path"), is(FTP_TEST_PATH_PREFIX + "02.csv"));
+    }
+
+    @Test
+    public void testListFilesByPrefixIncrementalFalse()
+    {
+        ConfigSource config = config()
+                .deepCopy()
+                .set("incremental", false);
+
+        ConfigDiff configDiff = runner.transaction(config, new Control());
+
+        assertThat(configDiff.toString(), is("{}"));
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testFtpFileInputByOpen() throws Exception
+    {
+        ConfigSource config = config();
+        PluginTask task = config().loadConfig(PluginTask.class);
+
+        runner.transaction(config, new Control());
+
+        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        method.setAccessible(true);
+        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
+        task.setFiles(fileList);
+
+        assertRecords(config, output);
+    }
+
+    private static List<TaskReport> emptyTaskReports(int taskCount)
+    {
+        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
+        for (int i = 0; i < taskCount; i++) {
+            reports.add(Exec.newTaskReport());
+        }
+        return reports.build();
+    }
+
+    private class Control
+            implements InputPlugin.Control
+    {
+        @Override
+        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+        {
+            List<TaskReport> reports = new ArrayList<>();
+            for (int i = 0; i < taskCount; i++) {
+                reports.add(runner.run(taskSource, schema, i, output));
+            }
+            return reports;
+        }
+    }
+
+    private ConfigSource config()
+    {
+        return Exec.newConfigSource()
+                .set("host", FTP_TEST_HOST)
+                .set("port", FTP_TEST_PORT)
+                .set("user", FTP_TEST_USER)
+                .set("password", FTP_TEST_PASSWORD)
+                .set("path_prefix", FTP_TEST_PATH_PREFIX)
+                .set("last_path", "")
+                .set("file_ext", ".csv")
+                .set("max_connection_retry", 3)
+                .set("ssl", false)
+                .set("ssl_verify", false)
+                .set("ssl_verify_hostname", false)
+                .set("ssl_trusted_ca_cert_data", FTP_TEST_SSL_TRUSTED_CA_CERT_DATA)
+                .set("parser", parserConfig(schemaConfig()));
+    }
+
+    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    {
+        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        builder.put("type", "csv");
+        builder.put("newline", "CRLF");
+        builder.put("delimiter", ",");
+        builder.put("quote", "\"");
+        builder.put("escape", "\"");
+        builder.put("trim_if_not_quoted", false);
+        builder.put("skip_header_lines", 1);
+        builder.put("allow_extra_columns", false);
+        builder.put("allow_optional_columns", false);
+        builder.put("columns", schemaConfig);
+        return builder.build();
+    }
+
+    private ImmutableList<Object> schemaConfig()
+    {
+        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        builder.add(ImmutableMap.of("name", "id", "type", "long"));
+        builder.add(ImmutableMap.of("name", "account", "type", "long"));
+        builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
+        builder.add(ImmutableMap.of("name", "purchase", "type", "timestamp", "format", "%Y%m%d"));
+        builder.add(ImmutableMap.of("name", "comment", "type", "string"));
+        return builder.build();
+    }
+
+    public SSLPlugins.SSLPluginConfig sslConfig(PluginTask task)
+    {
+        return SSLPlugins.configure(task);
+    }
+
+    private void assertRecords(ConfigSource config, MockPageOutput output)
+    {
+        List<Object[]> records = getRecords(config, output);
+        assertThat(records.size(), is(8));
+        {
+            Object[] record = records.get(0);
+            assertThat((long) record[0], is(1L));
+            assertThat((long) record[1], is(32864L));
+            assertThat(record[2].toString(), is("2015-01-27 19:23:49 UTC"));
+            assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
+            assertThat(record[4].toString(), is("embulk"));
+        }
+
+        {
+            Object[] record = records.get(1);
+            assertThat((long) record[0], is(2L));
+            assertThat((long) record[1], is(14824L));
+            assertThat(record[2].toString(), is("2015-01-27 19:01:23 UTC"));
+            assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
+            assertThat(record[4].toString(), is("embulk jruby"));
+        }
+    }
+
+    private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
+    {
+        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        return Pages.toObjects(schema, output.pages);
+    }
+
+    private static String getDirectory(String dir)
+    {
+        if (dir != null && !dir.endsWith("/")) {
+            dir = dir + "/";
+        }
+        if (dir.startsWith("/")) {
+            dir = dir.replaceFirst("/", "");
+        }
+        return dir;
+    }
 }

data/src/test/resources/sample_01.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

data/src/test/resources/sample_02.csv

@@ -0,0 +1,5 @@
+id,account,time,purchase,comment
+1,32864,2015-01-27 19:23:49,20150127,embulk
+2,14824,2015-01-27 19:01:23,20150127,embulk jruby
+3,27559,2015-01-28 02:20:02,20150128,"Embulk ""csv"" parser plugin"
+4,11270,2015-01-29 11:54:36,20150129,NULL

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-ftp
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.2.0
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-06-29 00:00:00.000000000 Z
+date: 2018-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -49,11 +49,13 @@ files:
 - lib/embulk/input/ftp.rb
 - src/main/java/org/embulk/input/FtpFileInputPlugin.java
 - src/test/java/org/embulk/input/TestFtpFileInputPlugin.java
+- src/test/resources/sample_01.csv
+- src/test/resources/sample_02.csv
 - classpath/bcpkix-jdk15on-1.52.jar
-- classpath/embulk-input-ftp-0.1.6.jar
+- classpath/embulk-util-ftp-0.2.0.jar
 - classpath/bcprov-jdk15on-1.52.jar
 - classpath/ftp4j-1.7.2.jar
-- classpath/embulk-util-ftp-0.1.6.jar
+- classpath/embulk-input-ftp-0.2.0.jar
 homepage: https://github.com/embulk/embulk-input-ftp
 licenses:
 - Apache 2.0