embulk-input-ftp 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 27845469df3d11bb23cf8ef4a815733383a34768
-  data.tar.gz: 391a4e30104f40bb79d34b1780863bbac9fc32f0
+  metadata.gz: 803a0b43863b8ec7aedb1de35984911526561704
+  data.tar.gz: decc88b75fa71e334b64ff8853c00be0ea1c8bcf
 SHA512:
-  metadata.gz: 43298f5fba9022feb85f0523f99e703f69553fb47c1f1e41a0a34c78584bb162ff84291de2117367e26dae290d865bc90122e806baa8a10b1a3833ea2a5f3d2a
-  data.tar.gz: 663e55ceb928b929b00e8c53566617932b45dcdda8d4cb81d1c2b5a67ba2ed99387fbdc1d6fcd03b6e82e89c6615bacd27eb5d561b536f4be789659b1bc3cfe6
+  metadata.gz: 04e1fa0d6a4f8e3c46e149a22fe6e621b7e7dbfab297a0fb2f2730738bf9395e75d628f7a7e247d3a3f4caac4cf66d4dacbc6cacd129a1fbb1a850ea4c73f557
+  data.tar.gz: 436aba43e3f26efdbd3d1fbd079e6c9ffb183ea881c4048bbc87196754ac9e241703e011cd1e01fd881d85a76e69afb34d12f2264a7ecc98a17831c49196530e

data/src/main/java/org/embulk/input/FtpFileInputPlugin.java

@@ -14,6 +14,8 @@ import it.sauronsoftware.ftp4j.FTPException;
 import it.sauronsoftware.ftp4j.FTPFile;
 import it.sauronsoftware.ftp4j.FTPIllegalReplyException;
 import it.sauronsoftware.ftp4j.FTPListParseException;
+
+import org.apache.commons.lang3.StringUtils;
 import org.embulk.config.Config;
 import org.embulk.config.ConfigDefault;
 import org.embulk.config.ConfigDiff;
@@ -46,6 +48,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.regex.Pattern;
 
 import static org.embulk.spi.util.RetryExecutor.retryExecutor;
 
@@ -67,6 +70,10 @@ public class FtpFileInputPlugin
         @ConfigDefault("null")
         Optional<String> getLastPath();
 
+        @Config("path_match_pattern")
+        @ConfigDefault("\".*\"")
+        String getPathMatchPattern();
+
         @Config("incremental")
         @ConfigDefault("true")
         boolean getIncremental();
@@ -113,14 +120,22 @@ public class FtpFileInputPlugin
     }
 
     @Override
-    public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
+    public ConfigDiff transaction(final ConfigSource config, final FileInputPlugin.Control control)
     {
-        PluginTask task = config.loadConfig(PluginTask.class);
+        final PluginTask task = config.loadConfig(PluginTask.class);
 
         task.setSSLConfig(SSLPlugins.configure(task));
 
+        String pattern = task.getPathMatchPattern();
+        // If pattern is empty then use default pattern
+        if (StringUtils.trim(pattern).isEmpty()) {
+            pattern = ".*";
+        }
+        // Create path match pattern
+        final Pattern pathMatchPattern = Pattern.compile(pattern);
+
         // list files recursively
-        List<String> files = listFiles(log, task);
+        final List<String> files = listFiles(log, task, pathMatchPattern);
         task.setFiles(files);
         log.info("Using files {}", files);
 
@@ -131,16 +146,16 @@ public class FtpFileInputPlugin
     }
 
     @Override
-    public ConfigDiff resume(TaskSource taskSource,
-            int taskCount,
-            FileInputPlugin.Control control)
+    public ConfigDiff resume(final TaskSource taskSource,
+            final int taskCount,
+            final FileInputPlugin.Control control)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
 
         control.run(taskSource, taskCount);
 
         // build next config
-        ConfigDiff configDiff = Exec.newConfigDiff();
+        final ConfigDiff configDiff = Exec.newConfigDiff();
 
         // last_path
         if (task.getIncremental()) {
@@ -151,7 +166,7 @@ public class FtpFileInputPlugin
                 }
             }
             else {
-                List<String> files = new ArrayList<String>(task.getFiles());
+                final List<String> files = new ArrayList<String>(task.getFiles());
                 Collections.sort(files);
                 configDiff.set("last_path", files.get(files.size() - 1));
             }
@@ -161,14 +176,14 @@ public class FtpFileInputPlugin
     }
 
     @Override
-    public void cleanup(TaskSource taskSource,
-            int taskCount,
-            List<TaskReport> successTaskReports)
+    public void cleanup(final TaskSource taskSource,
+            final int taskCount,
+            final List<TaskReport> successTaskReports)
     {
         // do nothing
     }
 
-    private static FTPClient newFTPClient(Logger log, PluginTask task)
+    private static FTPClient newFTPClient(final Logger log, final PluginTask task)
     {
         FTPClient client = new FTPClient();
         try {
@@ -186,14 +201,14 @@ public class FtpFileInputPlugin
                     log.info("Using FTPS(FTPS/implicit) mode");
                 }
             }
-            int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
+            final int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
 
             client.addCommunicationListener(new LoggingCommunicationListner(log));
 
             // TODO configurable timeout parameters
             client.setAutoNoopTimeout(3000);
 
-            FTPConnector con = client.getConnector();
+            final FTPConnector con = client.getConnector();
             con.setConnectionTimeout(30);
             con.setReadTimeout(60);
             con.setCloseTimeout(60);
@@ -230,19 +245,19 @@ public class FtpFileInputPlugin
                 client.setCompressionEnabled(true);
             }
 
-            FTPClient connected = client;
+            final FTPClient connected = client;
             client = null;
             return connected;
         }
-        catch (FTPException ex) {
+        catch (final FTPException ex) {
             log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
             throw Throwables.propagate(ex);
         }
-        catch (FTPIllegalReplyException ex) {
+        catch (final FTPIllegalReplyException ex) {
             log.info("FTP protocol error");
             throw Throwables.propagate(ex);
         }
-        catch (IOException ex) {
+        catch (final IOException ex) {
             log.info("FTP network error: " + ex);
             throw Throwables.propagate(ex);
         }
@@ -253,37 +268,37 @@ public class FtpFileInputPlugin
         }
     }
 
-    static void disconnectClient(FTPClient client)
+    static void disconnectClient(final FTPClient client)
     {
         if (client.isConnected()) {
             try {
                 client.disconnect(false);
             }
-            catch (FTPException ex) {
+            catch (final FTPException ex) {
                 // do nothing
             }
-            catch (FTPIllegalReplyException ex) {
+            catch (final FTPIllegalReplyException ex) {
                 // do nothing
             }
-            catch (IOException ex) {
+            catch (final IOException ex) {
                 // do nothing
             }
         }
     }
 
-    private List<String> listFiles(Logger log, PluginTask task)
+    private List<String> listFiles(final Logger log, final PluginTask task, final Pattern pathMatchPattern)
     {
-        FTPClient client = newFTPClient(log, task);
+        final FTPClient client = newFTPClient(log, task);
         try {
-            return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath());
+            return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath(), pathMatchPattern);
         }
         finally {
             disconnectClient(client);
         }
     }
 
-    public static List<String> listFilesByPrefix(Logger log, FTPClient client,
-            String prefix, Optional<String> lastPath)
+    public static List<String> listFilesByPrefix(final Logger log, final FTPClient client,
+            final String prefix, final Optional<String> lastPath, final Pattern pathMatchPattern)
     {
         String directory;
         String fileNamePrefix;
@@ -292,7 +307,7 @@ public class FtpFileInputPlugin
             fileNamePrefix = "";
         }
         else {
-            int pos = prefix.lastIndexOf("/");
+            final int pos = prefix.lastIndexOf("/");
             if (pos < 0) {
                 directory = "";
                 fileNamePrefix = prefix;
@@ -303,7 +318,7 @@ public class FtpFileInputPlugin
             }
         }
 
-        ImmutableList.Builder<String> builder = ImmutableList.builder();
+        final ImmutableList.Builder<String> builder = ImmutableList.builder();
 
         try {
             String currentDirectory = client.currentDirectory();
@@ -314,33 +329,33 @@ public class FtpFileInputPlugin
                 currentDirectory = directory;
             }
 
-            for (FTPFile file : client.list()) {
+            for (final FTPFile file : client.list()) {
                 if (file.getName().startsWith(fileNamePrefix)) {
-                    listFilesRecursive(client, currentDirectory, file, lastPath, builder);
+                    listFilesRecursive(client, currentDirectory, file, lastPath, builder, pathMatchPattern);
                 }
             }
         }
-        catch (FTPListParseException ex) {
+        catch (final FTPListParseException ex) {
             log.info("FTP listing files failed");
             throw Throwables.propagate(ex);
         }
-        catch (FTPAbortedException ex) {
+        catch (final FTPAbortedException ex) {
             log.info("FTP listing files failed");
             throw Throwables.propagate(ex);
         }
-        catch (FTPDataTransferException ex) {
+        catch (final FTPDataTransferException ex) {
             log.info("FTP data transfer failed");
             throw Throwables.propagate(ex);
         }
-        catch (FTPException ex) {
+        catch (final FTPException ex) {
             log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
             throw Throwables.propagate(ex);
         }
-        catch (FTPIllegalReplyException ex) {
+        catch (final FTPIllegalReplyException ex) {
             log.info("FTP protocol error");
             throw Throwables.propagate(ex);
         }
-        catch (IOException ex) {
+        catch (final IOException ex) {
             log.info("FTP network error: " + ex);
             throw Throwables.propagate(ex);
         }
@@ -348,15 +363,15 @@ public class FtpFileInputPlugin
         return builder.build();
     }
 
-    private static void listFilesRecursive(FTPClient client,
-            String baseDirectoryPath, FTPFile file, Optional<String> lastPath,
-            ImmutableList.Builder<String> builder)
+    private static void listFilesRecursive(final FTPClient client,
+            String baseDirectoryPath, final FTPFile file, final Optional<String> lastPath,
+            final ImmutableList.Builder<String> builder, final Pattern pathMatchPattern)
         throws IOException, FTPException, FTPIllegalReplyException, FTPDataTransferException, FTPAbortedException, FTPListParseException
     {
         if (!baseDirectoryPath.endsWith("/")) {
             baseDirectoryPath = baseDirectoryPath + "/";
         }
-        String path = baseDirectoryPath + file.getName();
+        final String path = baseDirectoryPath + file.getName();
 
         if (lastPath.isPresent() && path.compareTo(lastPath.get()) <= 0) {
             return;
@@ -364,12 +379,14 @@ public class FtpFileInputPlugin
 
         switch (file.getType()) {
         case FTPFile.TYPE_FILE:
-            builder.add(path);
+            if (pathMatchPattern.matcher(path).find()) {
+                builder.add(path);
+            }
             break;
         case FTPFile.TYPE_DIRECTORY:
             client.changeDirectory(path);
-            for (FTPFile subFile : client.list()) {
-                listFilesRecursive(client, path, subFile, lastPath, builder);
+            for (final FTPFile subFile : client.list()) {
+                listFilesRecursive(client, path, subFile, lastPath, builder, pathMatchPattern);
             }
             client.changeDirectory(baseDirectoryPath);
             break;
@@ -379,9 +396,9 @@ public class FtpFileInputPlugin
     }
 
     @Override
-    public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
+    public TransactionalFileInput open(final TaskSource taskSource, final int taskIndex)
     {
-        PluginTask task = taskSource.loadTask(PluginTask.class);
+        final PluginTask task = taskSource.loadTask(PluginTask.class);
         return new FtpFileInput(log, task, taskIndex);
     }
 
@@ -390,17 +407,19 @@ public class FtpFileInputPlugin
     {
         private final Logger log;
 
-        public LoggingCommunicationListner(Logger log)
+        public LoggingCommunicationListner(final Logger log)
         {
             this.log = log;
         }
 
-        public void received(String statement)
+        @Override
+        public void received(final String statement)
         {
             log.info("< " + statement);
         }
 
-        public void sent(String statement)
+        @Override
+        public void sent(final String statement)
         {
             if (statement.startsWith("PASS")) {
                 // don't show password
@@ -419,19 +438,21 @@ public class FtpFileInputPlugin
         private long totalTransfer;
         private long nextTransferNotice;
 
-        public LoggingTransferListener(Logger log, long transferNoticeBytes)
+        public LoggingTransferListener(final Logger log, final long transferNoticeBytes)
         {
             this.log = log;
             this.transferNoticeBytes = transferNoticeBytes;
             this.nextTransferNotice = transferNoticeBytes;
         }
 
+        @Override
         public void started()
         {
             log.info("Transfer started");
         }
 
-        public void transferred(int length)
+        @Override
+        public void transferred(final int length)
         {
             totalTransfer += length;
             if (totalTransfer > nextTransferNotice) {
@@ -440,16 +461,19 @@ public class FtpFileInputPlugin
             }
         }
 
+        @Override
         public void completed()
         {
             log.info("Transfer completed " + totalTransfer + " bytes");
         }
 
+        @Override
         public void aborted()
         {
             log.info("Transfer aborted");
         }
 
+        @Override
         public void failed()
         {
             log.info("Transfer failed");
@@ -459,43 +483,45 @@ public class FtpFileInputPlugin
     private static final long TRANSFER_NOTICE_BYTES = 100 * 1024 * 1024;
 
     private static InputStream startDownload(final Logger log, final FTPClient client,
-            final String path, final long offset, ExecutorService executor)
+            final String path, final long offset, final ExecutorService executor)
     {
-        BlockingTransfer t = BlockingTransfer.submit(executor,
+        final BlockingTransfer t = BlockingTransfer.submit(executor,
                 new Function<BlockingTransfer, Runnable>()
                 {
+                    @Override
                     public Runnable apply(final BlockingTransfer transfer)
                     {
                         return new Runnable() {
+                            @Override
                             public void run()
                             {
                                 try {
                                     client.download(path, Channels.newOutputStream(transfer.getWriterChannel()), offset, new LoggingTransferListener(log, TRANSFER_NOTICE_BYTES));
                                 }
-                                catch (FTPException ex) {
+                                catch (final FTPException ex) {
                                     log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
                                     throw Throwables.propagate(ex);
                                 }
-                                catch (FTPDataTransferException ex) {
+                                catch (final FTPDataTransferException ex) {
                                     log.info("FTP data transfer failed");
                                     throw Throwables.propagate(ex);
                                 }
-                                catch (FTPAbortedException ex) {
+                                catch (final FTPAbortedException ex) {
                                     log.info("FTP listing files failed");
                                     throw Throwables.propagate(ex);
                                 }
-                                catch (FTPIllegalReplyException ex) {
+                                catch (final FTPIllegalReplyException ex) {
                                     log.info("FTP protocol error");
                                     throw Throwables.propagate(ex);
                                 }
-                                catch (IOException ex) {
+                                catch (final IOException ex) {
                                     throw Throwables.propagate(ex);
                                 }
                                 finally {
                                     try {
                                         transfer.getWriterChannel().close();
                                     }
-                                    catch (IOException ex) {
+                                    catch (final IOException ex) {
                                         throw new RuntimeException(ex);
                                     }
                                 }
@@ -514,7 +540,7 @@ public class FtpFileInputPlugin
         private final ExecutorService executor;
         private final String path;
 
-        public FtpInputStreamReopener(Logger log, FTPClient client, ExecutorService executor, String path)
+        public FtpInputStreamReopener(final Logger log, final FTPClient client, final ExecutorService executor, final String path)
         {
             this.log = log;
             this.client = client;
@@ -539,16 +565,16 @@ public class FtpFileInputPlugin
                         }
 
                         @Override
-                        public boolean isRetryableException(Exception exception)
+                        public boolean isRetryableException(final Exception exception)
                         {
                             return true;  // TODO
                         }
 
                         @Override
-                        public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
+                        public void onRetry(final Exception exception, final int retryCount, final int retryLimit, final int retryWait)
                                 throws RetryGiveupException
                         {
-                            String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
+                            final String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
                                     retryCount, retryLimit, retryWait / 1000, exception.getMessage());
                             if (retryCount % 3 == 0) {
                                 log.warn(message, exception);
@@ -559,17 +585,17 @@ public class FtpFileInputPlugin
                         }
 
                         @Override
-                        public void onGiveup(Exception firstException, Exception lastException)
+                        public void onGiveup(final Exception firstException, final Exception lastException)
                                 throws RetryGiveupException
                         {
                         }
                     });
             }
-            catch (RetryGiveupException ex) {
+            catch (final RetryGiveupException ex) {
                 Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
                 throw Throwables.propagate(ex.getCause());
             }
-            catch (InterruptedException ex) {
+            catch (final InterruptedException ex) {
                 throw new InterruptedIOException();
             }
         }
@@ -585,7 +611,7 @@ public class FtpFileInputPlugin
         private final String path;
         private boolean opened = false;
 
-        public SingleFileProvider(Logger log, PluginTask task, int taskIndex)
+        public SingleFileProvider(final Logger log, final PluginTask task, final int taskIndex)
         {
             this.log = log;
             this.client = newFTPClient(log, task);
@@ -628,15 +654,17 @@ public class FtpFileInputPlugin
             extends InputStreamFileInput
             implements TransactionalFileInput
     {
-        public FtpFileInput(Logger log, PluginTask task, int taskIndex)
+        public FtpFileInput(final Logger log, final PluginTask task, final int taskIndex)
         {
             super(task.getBufferAllocator(), new SingleFileProvider(log, task, taskIndex));
         }
 
+        @Override
         public void abort()
         {
         }
 
+        @Override
         public TaskReport commit()
         {
             return Exec.newTaskReport();

data/src/test/java/org/embulk/input/TestFtpFileInputPlugin.java

@@ -19,6 +19,7 @@ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
 import org.embulk.spi.util.Pages;
 import org.embulk.standards.CsvParserPlugin;
 import org.embulk.util.ftp.SSLPlugins;
+import org.junit.Assert;
 import org.junit.Before;
 import org.junit.BeforeClass;
 import org.junit.Rule;
@@ -30,9 +31,9 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.regex.Pattern;
 
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertEquals;
 
 public class TestFtpFileInputPlugin
 {
@@ -45,8 +46,10 @@ public class TestFtpFileInputPlugin
     private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
     private static String FTP_TEST_DIRECTORY;
     private static String FTP_TEST_PATH_PREFIX;
+
     private FileInputRunner runner;
     private TestPageBuilderReader.MockPageOutput output;
+    private final Pattern defaultPathMatchPattern = Pattern.compile(".*");
 
     /*
      * This test case requires environment variables
@@ -86,7 +89,7 @@ public class TestFtpFileInputPlugin
     @Test(expected = RuntimeException.class) // TODO ConfigException should be thrown
     public void testTransactionWithInvalidHost()
     {
-        ConfigSource config = config().deepCopy()
+        final ConfigSource config = config().deepCopy()
                 .set("host", "non-exists.example.com");
 
         runner.transaction(config, new Control());
@@ -95,24 +98,24 @@ public class TestFtpFileInputPlugin
     @Test
     public void testResume()
     {
-        PluginTask task = config().loadConfig(PluginTask.class);
+        final PluginTask task = config().loadConfig(PluginTask.class);
         task.setSSLConfig(sslConfig(task));
         task.setFiles(Arrays.asList("in/aa/a"));
-        ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
+        final ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
         {
             @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
             {
                 return emptyTaskReports(taskCount);
             }
         });
-        assertThat(configDiff.get(String.class, "last_path"), is("in/aa/a"));
+        assertEquals(configDiff.get(String.class, "last_path"), "in/aa/a");
     }
 
     @Test
     public void testCleanup()
     {
-        PluginTask task = config().loadConfig(PluginTask.class);
+        final PluginTask task = config().loadConfig(PluginTask.class);
         plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
     }
 
@@ -120,87 +123,237 @@ public class TestFtpFileInputPlugin
     @SuppressWarnings("unchecked")
     public void testListFilesWithNonExistPath() throws Exception
     {
-        ConfigSource config = config().deepCopy()
+        final ConfigSource config = config().deepCopy()
                 .set("path_prefix", "non-exists-path");
-        PluginTask task = config.loadConfig(PluginTask.class);
+        final PluginTask task = config.loadConfig(PluginTask.class);
         plugin.transaction(config, new FileInputPlugin.Control() {
             @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
             {
-                assertThat(taskCount, is(0));
+                assertEquals(taskCount, 0);
                 return emptyTaskReports(taskCount);
             }
         });
 
-        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
         method.setAccessible(true);
-        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
-        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
-        assertThat(fileList.size(), is(0));
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
+        assertEquals(fileList.size(), 0);
     }
 
     @Test
     @SuppressWarnings("unchecked")
     public void testListFiles() throws Exception
     {
-        List<String> expected = Arrays.asList(
+        final List<String> expected = Arrays.asList(
+            FTP_TEST_PATH_PREFIX + "01.csv",
+            FTP_TEST_PATH_PREFIX + "02.csv"
+        );
+
+        final ConfigSource config = config();
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
+            {
+                assertEquals(taskCount, 2);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
+        method.setAccessible(true);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
+
+        assertEquals(fileList.get(0), expected.get(0));
+        assertEquals(fileList.get(1), expected.get(1));
+        assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFileWithEmptyPattern() throws Exception
+    {
+        final List<String> expected = Arrays.asList(
             FTP_TEST_PATH_PREFIX + "01.csv",
             FTP_TEST_PATH_PREFIX + "02.csv"
         );
 
-        ConfigSource config = config();
+        final ConfigSource config = config();
+        config.set("path_match_pattern", "");
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
+            {
+                assertEquals(taskCount, 2);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
+        method.setAccessible(true);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
+
+        assertEquals(fileList.get(0), expected.get(0));
+        assertEquals(fileList.get(1), expected.get(1));
+        assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFileWithSpacesPattern() throws Exception
+    {
+        final List<String> expected = Arrays.asList(
+            FTP_TEST_PATH_PREFIX + "01.csv",
+            FTP_TEST_PATH_PREFIX + "02.csv"
+        );
+
+        final ConfigSource config = config();
+        config.set("path_match_pattern", "    ");
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
+            {
+                assertEquals(taskCount, 2);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
+        method.setAccessible(true);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
+
+        assertEquals(fileList.get(0), expected.get(0));
+        assertEquals(fileList.get(1), expected.get(1));
+        assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFilesWithPathMatcher() throws Exception
+    {
+        final String pattern = FTP_TEST_PATH_PREFIX + "02";
+        final Pattern pathMatchPattern = Pattern.compile(pattern);
+        final List<String> expected = Arrays.asList(
+            FTP_TEST_PATH_PREFIX + "02.csv"
+        );
+
+        final ConfigSource config = config();
+        config.set("path_match_pattern", pattern);
         final PluginTask task = config.loadConfig(PluginTask.class);
-        ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
             @Override
-            public List<TaskReport> run(TaskSource taskSource, int taskCount)
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
             {
-                assertThat(taskCount, is(2));
+                assertEquals(taskCount, 1);
                 return emptyTaskReports(taskCount);
             }
         });
 
-        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
         method.setAccessible(true);
-        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
-        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
-        assertThat(fileList.get(0), is(expected.get(0)));
-        assertThat(fileList.get(1), is(expected.get(1)));
-        assertThat(configDiff.get(String.class, "last_path"), is(FTP_TEST_PATH_PREFIX + "02.csv"));
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
+
+        assertEquals(fileList.get(0), expected.get(0));
+        assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFilesWithNonExistPathMatcher() throws Exception
+    {
+        final String pattern = "non_exist_file_matcher";
+        final Pattern pathMatchPattern = Pattern.compile(pattern);
+
+        final ConfigSource config = config();
+        config.set("path_match_pattern", pattern);
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
+            {
+                assertEquals(taskCount, 0);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
+        method.setAccessible(true);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
+
+        assertEquals(fileList.size(), 0);
+        assertEquals(configDiff.get(String.class, "last_path"), "");
+    }
+
+    @Test
+    @SuppressWarnings("unchecked")
+    public void testListFilesWithNotExactPathMatcher() throws Exception
+    {
+        final String pattern = "\\b" + FTP_TEST_PATH_PREFIX + "\\b";
+        final Pattern pathMatchPattern = Pattern.compile(pattern);
+
+        final ConfigSource config = config();
+        config.set("path_match_pattern", pattern);
+        final PluginTask task = config.loadConfig(PluginTask.class);
+        final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
+            @Override
+            public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
+            {
+                assertEquals(taskCount, 0);
+                return emptyTaskReports(taskCount);
+            }
+        });
+
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
+        method.setAccessible(true);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
+
+        assertEquals(fileList.size(), 0);
+        assertEquals(configDiff.get(String.class, "last_path"), "");
     }
 
     @Test
     public void testListFilesByPrefixIncrementalFalse()
     {
-        ConfigSource config = config()
+        final ConfigSource config = config()
                 .deepCopy()
                 .set("incremental", false);
 
-        ConfigDiff configDiff = runner.transaction(config, new Control());
+        final ConfigDiff configDiff = runner.transaction(config, new Control());
 
-        assertThat(configDiff.toString(), is("{}"));
+        Assert.assertEquals(configDiff.toString(), "{}");
     }
 
     @Test
     @SuppressWarnings("unchecked")
     public void testFtpFileInputByOpen() throws Exception
     {
-        ConfigSource config = config();
-        PluginTask task = config().loadConfig(PluginTask.class);
+        final ConfigSource config = config();
+        final PluginTask task = config().loadConfig(PluginTask.class);
 
         runner.transaction(config, new Control());
 
-        Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
+        final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
         method.setAccessible(true);
-        Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
-        List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
+        final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
+        final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
         task.setFiles(fileList);
 
         assertRecords(config, output);
     }
 
-    private static List<TaskReport> emptyTaskReports(int taskCount)
+    private static List<TaskReport> emptyTaskReports(final int taskCount)
     {
-        ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
+        final ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
         for (int i = 0; i < taskCount; i++) {
             reports.add(Exec.newTaskReport());
         }
@@ -211,9 +364,9 @@ public class TestFtpFileInputPlugin
             implements InputPlugin.Control
     {
         @Override
-        public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
+        public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
         {
-            List<TaskReport> reports = new ArrayList<>();
+            final List<TaskReport> reports = new ArrayList<>();
             for (int i = 0; i < taskCount; i++) {
                 reports.add(runner.run(taskSource, schema, i, output));
             }
@@ -239,9 +392,9 @@ public class TestFtpFileInputPlugin
                 .set("parser", parserConfig(schemaConfig()));
     }
 
-    private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
+    private ImmutableMap<String, Object> parserConfig(final ImmutableList<Object> schemaConfig)
     {
-        ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
+        final ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
         builder.put("type", "csv");
         builder.put("newline", "CRLF");
         builder.put("delimiter", ",");
@@ -257,7 +410,7 @@ public class TestFtpFileInputPlugin
 
     private ImmutableList<Object> schemaConfig()
     {
-        ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
+        final ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
         builder.add(ImmutableMap.of("name", "id", "type", "long"));
         builder.add(ImmutableMap.of("name", "account", "type", "long"));
         builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
@@ -266,37 +419,37 @@ public class TestFtpFileInputPlugin
         return builder.build();
     }
 
-    public SSLPlugins.SSLPluginConfig sslConfig(PluginTask task)
+    public SSLPlugins.SSLPluginConfig sslConfig(final PluginTask task)
     {
         return SSLPlugins.configure(task);
     }
 
-    private void assertRecords(ConfigSource config, MockPageOutput output)
+    private void assertRecords(final ConfigSource config, final MockPageOutput output)
     {
-        List<Object[]> records = getRecords(config, output);
-        assertThat(records.size(), is(8));
+        final List<Object[]> records = getRecords(config, output);
+        assertEquals(records.size(), 8);
         {
-            Object[] record = records.get(0);
-            assertThat((long) record[0], is(1L));
-            assertThat((long) record[1], is(32864L));
-            assertThat(record[2].toString(), is("2015-01-27 19:23:49 UTC"));
-            assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
-            assertThat(record[4].toString(), is("embulk"));
+            final Object[] record = records.get(0);
+            assertEquals((long) record[0], 1L);
+            assertEquals((long) record[1], 32864L);
+            assertEquals(record[2].toString(), "2015-01-27 19:23:49 UTC");
+            assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
+            assertEquals(record[4].toString(), "embulk");
         }
 
         {
-            Object[] record = records.get(1);
-            assertThat((long) record[0], is(2L));
-            assertThat((long) record[1], is(14824L));
-            assertThat(record[2].toString(), is("2015-01-27 19:01:23 UTC"));
-            assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
-            assertThat(record[4].toString(), is("embulk jruby"));
+            final Object[] record = records.get(1);
+            assertEquals((long) record[0], 2L);
+            assertEquals((long) record[1], 14824L);
+            assertEquals(record[2].toString(), "2015-01-27 19:01:23 UTC");
+            assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
+            assertEquals(record[4].toString(), "embulk jruby");
         }
     }
 
-    private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
+    private List<Object[]> getRecords(final ConfigSource config, final MockPageOutput output)
     {
-        Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
+        final Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
         return Pages.toObjects(schema, output.pages);
     }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-ftp
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-12-26 00:00:00.000000000 Z
+date: 2019-11-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -52,10 +52,10 @@ files:
 - src/test/resources/sample_01.csv
 - src/test/resources/sample_02.csv
 - classpath/bcpkix-jdk15on-1.52.jar
-- classpath/embulk-util-ftp-0.2.0.jar
+- classpath/embulk-util-ftp-0.2.1.jar
 - classpath/bcprov-jdk15on-1.52.jar
 - classpath/ftp4j-1.7.2.jar
-- classpath/embulk-input-ftp-0.2.0.jar
+- classpath/embulk-input-ftp-0.2.1.jar
 homepage: https://github.com/embulk/embulk-input-ftp
 licenses:
 - Apache 2.0