embulk-input-ftp 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 27845469df3d11bb23cf8ef4a815733383a34768
4
- data.tar.gz: 391a4e30104f40bb79d34b1780863bbac9fc32f0
3
+ metadata.gz: 803a0b43863b8ec7aedb1de35984911526561704
4
+ data.tar.gz: decc88b75fa71e334b64ff8853c00be0ea1c8bcf
5
5
  SHA512:
6
- metadata.gz: 43298f5fba9022feb85f0523f99e703f69553fb47c1f1e41a0a34c78584bb162ff84291de2117367e26dae290d865bc90122e806baa8a10b1a3833ea2a5f3d2a
7
- data.tar.gz: 663e55ceb928b929b00e8c53566617932b45dcdda8d4cb81d1c2b5a67ba2ed99387fbdc1d6fcd03b6e82e89c6615bacd27eb5d561b536f4be789659b1bc3cfe6
6
+ metadata.gz: 04e1fa0d6a4f8e3c46e149a22fe6e621b7e7dbfab297a0fb2f2730738bf9395e75d628f7a7e247d3a3f4caac4cf66d4dacbc6cacd129a1fbb1a850ea4c73f557
7
+ data.tar.gz: 436aba43e3f26efdbd3d1fbd079e6c9ffb183ea881c4048bbc87196754ac9e241703e011cd1e01fd881d85a76e69afb34d12f2264a7ecc98a17831c49196530e
@@ -14,6 +14,8 @@ import it.sauronsoftware.ftp4j.FTPException;
14
14
  import it.sauronsoftware.ftp4j.FTPFile;
15
15
  import it.sauronsoftware.ftp4j.FTPIllegalReplyException;
16
16
  import it.sauronsoftware.ftp4j.FTPListParseException;
17
+
18
+ import org.apache.commons.lang3.StringUtils;
17
19
  import org.embulk.config.Config;
18
20
  import org.embulk.config.ConfigDefault;
19
21
  import org.embulk.config.ConfigDiff;
@@ -46,6 +48,7 @@ import java.util.List;
46
48
  import java.util.Optional;
47
49
  import java.util.concurrent.ExecutorService;
48
50
  import java.util.concurrent.Executors;
51
+ import java.util.regex.Pattern;
49
52
 
50
53
  import static org.embulk.spi.util.RetryExecutor.retryExecutor;
51
54
 
@@ -67,6 +70,10 @@ public class FtpFileInputPlugin
67
70
  @ConfigDefault("null")
68
71
  Optional<String> getLastPath();
69
72
 
73
+ @Config("path_match_pattern")
74
+ @ConfigDefault("\".*\"")
75
+ String getPathMatchPattern();
76
+
70
77
  @Config("incremental")
71
78
  @ConfigDefault("true")
72
79
  boolean getIncremental();
@@ -113,14 +120,22 @@ public class FtpFileInputPlugin
113
120
  }
114
121
 
115
122
  @Override
116
- public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
123
+ public ConfigDiff transaction(final ConfigSource config, final FileInputPlugin.Control control)
117
124
  {
118
- PluginTask task = config.loadConfig(PluginTask.class);
125
+ final PluginTask task = config.loadConfig(PluginTask.class);
119
126
 
120
127
  task.setSSLConfig(SSLPlugins.configure(task));
121
128
 
129
+ String pattern = task.getPathMatchPattern();
130
+ // If pattern is empty then use default pattern
131
+ if (StringUtils.trim(pattern).isEmpty()) {
132
+ pattern = ".*";
133
+ }
134
+ // Create path match pattern
135
+ final Pattern pathMatchPattern = Pattern.compile(pattern);
136
+
122
137
  // list files recursively
123
- List<String> files = listFiles(log, task);
138
+ final List<String> files = listFiles(log, task, pathMatchPattern);
124
139
  task.setFiles(files);
125
140
  log.info("Using files {}", files);
126
141
 
@@ -131,16 +146,16 @@ public class FtpFileInputPlugin
131
146
  }
132
147
 
133
148
  @Override
134
- public ConfigDiff resume(TaskSource taskSource,
135
- int taskCount,
136
- FileInputPlugin.Control control)
149
+ public ConfigDiff resume(final TaskSource taskSource,
150
+ final int taskCount,
151
+ final FileInputPlugin.Control control)
137
152
  {
138
- PluginTask task = taskSource.loadTask(PluginTask.class);
153
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
139
154
 
140
155
  control.run(taskSource, taskCount);
141
156
 
142
157
  // build next config
143
- ConfigDiff configDiff = Exec.newConfigDiff();
158
+ final ConfigDiff configDiff = Exec.newConfigDiff();
144
159
 
145
160
  // last_path
146
161
  if (task.getIncremental()) {
@@ -151,7 +166,7 @@ public class FtpFileInputPlugin
151
166
  }
152
167
  }
153
168
  else {
154
- List<String> files = new ArrayList<String>(task.getFiles());
169
+ final List<String> files = new ArrayList<String>(task.getFiles());
155
170
  Collections.sort(files);
156
171
  configDiff.set("last_path", files.get(files.size() - 1));
157
172
  }
@@ -161,14 +176,14 @@ public class FtpFileInputPlugin
161
176
  }
162
177
 
163
178
  @Override
164
- public void cleanup(TaskSource taskSource,
165
- int taskCount,
166
- List<TaskReport> successTaskReports)
179
+ public void cleanup(final TaskSource taskSource,
180
+ final int taskCount,
181
+ final List<TaskReport> successTaskReports)
167
182
  {
168
183
  // do nothing
169
184
  }
170
185
 
171
- private static FTPClient newFTPClient(Logger log, PluginTask task)
186
+ private static FTPClient newFTPClient(final Logger log, final PluginTask task)
172
187
  {
173
188
  FTPClient client = new FTPClient();
174
189
  try {
@@ -186,14 +201,14 @@ public class FtpFileInputPlugin
186
201
  log.info("Using FTPS(FTPS/implicit) mode");
187
202
  }
188
203
  }
189
- int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
204
+ final int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
190
205
 
191
206
  client.addCommunicationListener(new LoggingCommunicationListner(log));
192
207
 
193
208
  // TODO configurable timeout parameters
194
209
  client.setAutoNoopTimeout(3000);
195
210
 
196
- FTPConnector con = client.getConnector();
211
+ final FTPConnector con = client.getConnector();
197
212
  con.setConnectionTimeout(30);
198
213
  con.setReadTimeout(60);
199
214
  con.setCloseTimeout(60);
@@ -230,19 +245,19 @@ public class FtpFileInputPlugin
230
245
  client.setCompressionEnabled(true);
231
246
  }
232
247
 
233
- FTPClient connected = client;
248
+ final FTPClient connected = client;
234
249
  client = null;
235
250
  return connected;
236
251
  }
237
- catch (FTPException ex) {
252
+ catch (final FTPException ex) {
238
253
  log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
239
254
  throw Throwables.propagate(ex);
240
255
  }
241
- catch (FTPIllegalReplyException ex) {
256
+ catch (final FTPIllegalReplyException ex) {
242
257
  log.info("FTP protocol error");
243
258
  throw Throwables.propagate(ex);
244
259
  }
245
- catch (IOException ex) {
260
+ catch (final IOException ex) {
246
261
  log.info("FTP network error: " + ex);
247
262
  throw Throwables.propagate(ex);
248
263
  }
@@ -253,37 +268,37 @@ public class FtpFileInputPlugin
253
268
  }
254
269
  }
255
270
 
256
- static void disconnectClient(FTPClient client)
271
+ static void disconnectClient(final FTPClient client)
257
272
  {
258
273
  if (client.isConnected()) {
259
274
  try {
260
275
  client.disconnect(false);
261
276
  }
262
- catch (FTPException ex) {
277
+ catch (final FTPException ex) {
263
278
  // do nothing
264
279
  }
265
- catch (FTPIllegalReplyException ex) {
280
+ catch (final FTPIllegalReplyException ex) {
266
281
  // do nothing
267
282
  }
268
- catch (IOException ex) {
283
+ catch (final IOException ex) {
269
284
  // do nothing
270
285
  }
271
286
  }
272
287
  }
273
288
 
274
- private List<String> listFiles(Logger log, PluginTask task)
289
+ private List<String> listFiles(final Logger log, final PluginTask task, final Pattern pathMatchPattern)
275
290
  {
276
- FTPClient client = newFTPClient(log, task);
291
+ final FTPClient client = newFTPClient(log, task);
277
292
  try {
278
- return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath());
293
+ return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath(), pathMatchPattern);
279
294
  }
280
295
  finally {
281
296
  disconnectClient(client);
282
297
  }
283
298
  }
284
299
 
285
- public static List<String> listFilesByPrefix(Logger log, FTPClient client,
286
- String prefix, Optional<String> lastPath)
300
+ public static List<String> listFilesByPrefix(final Logger log, final FTPClient client,
301
+ final String prefix, final Optional<String> lastPath, final Pattern pathMatchPattern)
287
302
  {
288
303
  String directory;
289
304
  String fileNamePrefix;
@@ -292,7 +307,7 @@ public class FtpFileInputPlugin
292
307
  fileNamePrefix = "";
293
308
  }
294
309
  else {
295
- int pos = prefix.lastIndexOf("/");
310
+ final int pos = prefix.lastIndexOf("/");
296
311
  if (pos < 0) {
297
312
  directory = "";
298
313
  fileNamePrefix = prefix;
@@ -303,7 +318,7 @@ public class FtpFileInputPlugin
303
318
  }
304
319
  }
305
320
 
306
- ImmutableList.Builder<String> builder = ImmutableList.builder();
321
+ final ImmutableList.Builder<String> builder = ImmutableList.builder();
307
322
 
308
323
  try {
309
324
  String currentDirectory = client.currentDirectory();
@@ -314,33 +329,33 @@ public class FtpFileInputPlugin
314
329
  currentDirectory = directory;
315
330
  }
316
331
 
317
- for (FTPFile file : client.list()) {
332
+ for (final FTPFile file : client.list()) {
318
333
  if (file.getName().startsWith(fileNamePrefix)) {
319
- listFilesRecursive(client, currentDirectory, file, lastPath, builder);
334
+ listFilesRecursive(client, currentDirectory, file, lastPath, builder, pathMatchPattern);
320
335
  }
321
336
  }
322
337
  }
323
- catch (FTPListParseException ex) {
338
+ catch (final FTPListParseException ex) {
324
339
  log.info("FTP listing files failed");
325
340
  throw Throwables.propagate(ex);
326
341
  }
327
- catch (FTPAbortedException ex) {
342
+ catch (final FTPAbortedException ex) {
328
343
  log.info("FTP listing files failed");
329
344
  throw Throwables.propagate(ex);
330
345
  }
331
- catch (FTPDataTransferException ex) {
346
+ catch (final FTPDataTransferException ex) {
332
347
  log.info("FTP data transfer failed");
333
348
  throw Throwables.propagate(ex);
334
349
  }
335
- catch (FTPException ex) {
350
+ catch (final FTPException ex) {
336
351
  log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
337
352
  throw Throwables.propagate(ex);
338
353
  }
339
- catch (FTPIllegalReplyException ex) {
354
+ catch (final FTPIllegalReplyException ex) {
340
355
  log.info("FTP protocol error");
341
356
  throw Throwables.propagate(ex);
342
357
  }
343
- catch (IOException ex) {
358
+ catch (final IOException ex) {
344
359
  log.info("FTP network error: " + ex);
345
360
  throw Throwables.propagate(ex);
346
361
  }
@@ -348,15 +363,15 @@ public class FtpFileInputPlugin
348
363
  return builder.build();
349
364
  }
350
365
 
351
- private static void listFilesRecursive(FTPClient client,
352
- String baseDirectoryPath, FTPFile file, Optional<String> lastPath,
353
- ImmutableList.Builder<String> builder)
366
+ private static void listFilesRecursive(final FTPClient client,
367
+ String baseDirectoryPath, final FTPFile file, final Optional<String> lastPath,
368
+ final ImmutableList.Builder<String> builder, final Pattern pathMatchPattern)
354
369
  throws IOException, FTPException, FTPIllegalReplyException, FTPDataTransferException, FTPAbortedException, FTPListParseException
355
370
  {
356
371
  if (!baseDirectoryPath.endsWith("/")) {
357
372
  baseDirectoryPath = baseDirectoryPath + "/";
358
373
  }
359
- String path = baseDirectoryPath + file.getName();
374
+ final String path = baseDirectoryPath + file.getName();
360
375
 
361
376
  if (lastPath.isPresent() && path.compareTo(lastPath.get()) <= 0) {
362
377
  return;
@@ -364,12 +379,14 @@ public class FtpFileInputPlugin
364
379
 
365
380
  switch (file.getType()) {
366
381
  case FTPFile.TYPE_FILE:
367
- builder.add(path);
382
+ if (pathMatchPattern.matcher(path).find()) {
383
+ builder.add(path);
384
+ }
368
385
  break;
369
386
  case FTPFile.TYPE_DIRECTORY:
370
387
  client.changeDirectory(path);
371
- for (FTPFile subFile : client.list()) {
372
- listFilesRecursive(client, path, subFile, lastPath, builder);
388
+ for (final FTPFile subFile : client.list()) {
389
+ listFilesRecursive(client, path, subFile, lastPath, builder, pathMatchPattern);
373
390
  }
374
391
  client.changeDirectory(baseDirectoryPath);
375
392
  break;
@@ -379,9 +396,9 @@ public class FtpFileInputPlugin
379
396
  }
380
397
 
381
398
  @Override
382
- public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
399
+ public TransactionalFileInput open(final TaskSource taskSource, final int taskIndex)
383
400
  {
384
- PluginTask task = taskSource.loadTask(PluginTask.class);
401
+ final PluginTask task = taskSource.loadTask(PluginTask.class);
385
402
  return new FtpFileInput(log, task, taskIndex);
386
403
  }
387
404
 
@@ -390,17 +407,19 @@ public class FtpFileInputPlugin
390
407
  {
391
408
  private final Logger log;
392
409
 
393
- public LoggingCommunicationListner(Logger log)
410
+ public LoggingCommunicationListner(final Logger log)
394
411
  {
395
412
  this.log = log;
396
413
  }
397
414
 
398
- public void received(String statement)
415
+ @Override
416
+ public void received(final String statement)
399
417
  {
400
418
  log.info("< " + statement);
401
419
  }
402
420
 
403
- public void sent(String statement)
421
+ @Override
422
+ public void sent(final String statement)
404
423
  {
405
424
  if (statement.startsWith("PASS")) {
406
425
  // don't show password
@@ -419,19 +438,21 @@ public class FtpFileInputPlugin
419
438
  private long totalTransfer;
420
439
  private long nextTransferNotice;
421
440
 
422
- public LoggingTransferListener(Logger log, long transferNoticeBytes)
441
+ public LoggingTransferListener(final Logger log, final long transferNoticeBytes)
423
442
  {
424
443
  this.log = log;
425
444
  this.transferNoticeBytes = transferNoticeBytes;
426
445
  this.nextTransferNotice = transferNoticeBytes;
427
446
  }
428
447
 
448
+ @Override
429
449
  public void started()
430
450
  {
431
451
  log.info("Transfer started");
432
452
  }
433
453
 
434
- public void transferred(int length)
454
+ @Override
455
+ public void transferred(final int length)
435
456
  {
436
457
  totalTransfer += length;
437
458
  if (totalTransfer > nextTransferNotice) {
@@ -440,16 +461,19 @@ public class FtpFileInputPlugin
440
461
  }
441
462
  }
442
463
 
464
+ @Override
443
465
  public void completed()
444
466
  {
445
467
  log.info("Transfer completed " + totalTransfer + " bytes");
446
468
  }
447
469
 
470
+ @Override
448
471
  public void aborted()
449
472
  {
450
473
  log.info("Transfer aborted");
451
474
  }
452
475
 
476
+ @Override
453
477
  public void failed()
454
478
  {
455
479
  log.info("Transfer failed");
@@ -459,43 +483,45 @@ public class FtpFileInputPlugin
459
483
  private static final long TRANSFER_NOTICE_BYTES = 100 * 1024 * 1024;
460
484
 
461
485
  private static InputStream startDownload(final Logger log, final FTPClient client,
462
- final String path, final long offset, ExecutorService executor)
486
+ final String path, final long offset, final ExecutorService executor)
463
487
  {
464
- BlockingTransfer t = BlockingTransfer.submit(executor,
488
+ final BlockingTransfer t = BlockingTransfer.submit(executor,
465
489
  new Function<BlockingTransfer, Runnable>()
466
490
  {
491
+ @Override
467
492
  public Runnable apply(final BlockingTransfer transfer)
468
493
  {
469
494
  return new Runnable() {
495
+ @Override
470
496
  public void run()
471
497
  {
472
498
  try {
473
499
  client.download(path, Channels.newOutputStream(transfer.getWriterChannel()), offset, new LoggingTransferListener(log, TRANSFER_NOTICE_BYTES));
474
500
  }
475
- catch (FTPException ex) {
501
+ catch (final FTPException ex) {
476
502
  log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
477
503
  throw Throwables.propagate(ex);
478
504
  }
479
- catch (FTPDataTransferException ex) {
505
+ catch (final FTPDataTransferException ex) {
480
506
  log.info("FTP data transfer failed");
481
507
  throw Throwables.propagate(ex);
482
508
  }
483
- catch (FTPAbortedException ex) {
509
+ catch (final FTPAbortedException ex) {
484
510
  log.info("FTP listing files failed");
485
511
  throw Throwables.propagate(ex);
486
512
  }
487
- catch (FTPIllegalReplyException ex) {
513
+ catch (final FTPIllegalReplyException ex) {
488
514
  log.info("FTP protocol error");
489
515
  throw Throwables.propagate(ex);
490
516
  }
491
- catch (IOException ex) {
517
+ catch (final IOException ex) {
492
518
  throw Throwables.propagate(ex);
493
519
  }
494
520
  finally {
495
521
  try {
496
522
  transfer.getWriterChannel().close();
497
523
  }
498
- catch (IOException ex) {
524
+ catch (final IOException ex) {
499
525
  throw new RuntimeException(ex);
500
526
  }
501
527
  }
@@ -514,7 +540,7 @@ public class FtpFileInputPlugin
514
540
  private final ExecutorService executor;
515
541
  private final String path;
516
542
 
517
- public FtpInputStreamReopener(Logger log, FTPClient client, ExecutorService executor, String path)
543
+ public FtpInputStreamReopener(final Logger log, final FTPClient client, final ExecutorService executor, final String path)
518
544
  {
519
545
  this.log = log;
520
546
  this.client = client;
@@ -539,16 +565,16 @@ public class FtpFileInputPlugin
539
565
  }
540
566
 
541
567
  @Override
542
- public boolean isRetryableException(Exception exception)
568
+ public boolean isRetryableException(final Exception exception)
543
569
  {
544
570
  return true; // TODO
545
571
  }
546
572
 
547
573
  @Override
548
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
574
+ public void onRetry(final Exception exception, final int retryCount, final int retryLimit, final int retryWait)
549
575
  throws RetryGiveupException
550
576
  {
551
- String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
577
+ final String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
552
578
  retryCount, retryLimit, retryWait / 1000, exception.getMessage());
553
579
  if (retryCount % 3 == 0) {
554
580
  log.warn(message, exception);
@@ -559,17 +585,17 @@ public class FtpFileInputPlugin
559
585
  }
560
586
 
561
587
  @Override
562
- public void onGiveup(Exception firstException, Exception lastException)
588
+ public void onGiveup(final Exception firstException, final Exception lastException)
563
589
  throws RetryGiveupException
564
590
  {
565
591
  }
566
592
  });
567
593
  }
568
- catch (RetryGiveupException ex) {
594
+ catch (final RetryGiveupException ex) {
569
595
  Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
570
596
  throw Throwables.propagate(ex.getCause());
571
597
  }
572
- catch (InterruptedException ex) {
598
+ catch (final InterruptedException ex) {
573
599
  throw new InterruptedIOException();
574
600
  }
575
601
  }
@@ -585,7 +611,7 @@ public class FtpFileInputPlugin
585
611
  private final String path;
586
612
  private boolean opened = false;
587
613
 
588
- public SingleFileProvider(Logger log, PluginTask task, int taskIndex)
614
+ public SingleFileProvider(final Logger log, final PluginTask task, final int taskIndex)
589
615
  {
590
616
  this.log = log;
591
617
  this.client = newFTPClient(log, task);
@@ -628,15 +654,17 @@ public class FtpFileInputPlugin
628
654
  extends InputStreamFileInput
629
655
  implements TransactionalFileInput
630
656
  {
631
- public FtpFileInput(Logger log, PluginTask task, int taskIndex)
657
+ public FtpFileInput(final Logger log, final PluginTask task, final int taskIndex)
632
658
  {
633
659
  super(task.getBufferAllocator(), new SingleFileProvider(log, task, taskIndex));
634
660
  }
635
661
 
662
+ @Override
636
663
  public void abort()
637
664
  {
638
665
  }
639
666
 
667
+ @Override
640
668
  public TaskReport commit()
641
669
  {
642
670
  return Exec.newTaskReport();
@@ -19,6 +19,7 @@ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
19
19
  import org.embulk.spi.util.Pages;
20
20
  import org.embulk.standards.CsvParserPlugin;
21
21
  import org.embulk.util.ftp.SSLPlugins;
22
+ import org.junit.Assert;
22
23
  import org.junit.Before;
23
24
  import org.junit.BeforeClass;
24
25
  import org.junit.Rule;
@@ -30,9 +31,9 @@ import java.util.ArrayList;
30
31
  import java.util.Arrays;
31
32
  import java.util.List;
32
33
  import java.util.Map;
34
+ import java.util.regex.Pattern;
33
35
 
34
- import static org.hamcrest.CoreMatchers.is;
35
- import static org.junit.Assert.assertThat;
36
+ import static org.junit.Assert.assertEquals;
36
37
 
37
38
  public class TestFtpFileInputPlugin
38
39
  {
@@ -45,8 +46,10 @@ public class TestFtpFileInputPlugin
45
46
  private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
46
47
  private static String FTP_TEST_DIRECTORY;
47
48
  private static String FTP_TEST_PATH_PREFIX;
49
+
48
50
  private FileInputRunner runner;
49
51
  private TestPageBuilderReader.MockPageOutput output;
52
+ private final Pattern defaultPathMatchPattern = Pattern.compile(".*");
50
53
 
51
54
  /*
52
55
  * This test case requires environment variables
@@ -86,7 +89,7 @@ public class TestFtpFileInputPlugin
86
89
  @Test(expected = RuntimeException.class) // TODO ConfigException should be thrown
87
90
  public void testTransactionWithInvalidHost()
88
91
  {
89
- ConfigSource config = config().deepCopy()
92
+ final ConfigSource config = config().deepCopy()
90
93
  .set("host", "non-exists.example.com");
91
94
 
92
95
  runner.transaction(config, new Control());
@@ -95,24 +98,24 @@ public class TestFtpFileInputPlugin
95
98
  @Test
96
99
  public void testResume()
97
100
  {
98
- PluginTask task = config().loadConfig(PluginTask.class);
101
+ final PluginTask task = config().loadConfig(PluginTask.class);
99
102
  task.setSSLConfig(sslConfig(task));
100
103
  task.setFiles(Arrays.asList("in/aa/a"));
101
- ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
104
+ final ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
102
105
  {
103
106
  @Override
104
- public List<TaskReport> run(TaskSource taskSource, int taskCount)
107
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
105
108
  {
106
109
  return emptyTaskReports(taskCount);
107
110
  }
108
111
  });
109
- assertThat(configDiff.get(String.class, "last_path"), is("in/aa/a"));
112
+ assertEquals(configDiff.get(String.class, "last_path"), "in/aa/a");
110
113
  }
111
114
 
112
115
  @Test
113
116
  public void testCleanup()
114
117
  {
115
- PluginTask task = config().loadConfig(PluginTask.class);
118
+ final PluginTask task = config().loadConfig(PluginTask.class);
116
119
  plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
117
120
  }
118
121
 
@@ -120,87 +123,237 @@ public class TestFtpFileInputPlugin
120
123
  @SuppressWarnings("unchecked")
121
124
  public void testListFilesWithNonExistPath() throws Exception
122
125
  {
123
- ConfigSource config = config().deepCopy()
126
+ final ConfigSource config = config().deepCopy()
124
127
  .set("path_prefix", "non-exists-path");
125
- PluginTask task = config.loadConfig(PluginTask.class);
128
+ final PluginTask task = config.loadConfig(PluginTask.class);
126
129
  plugin.transaction(config, new FileInputPlugin.Control() {
127
130
  @Override
128
- public List<TaskReport> run(TaskSource taskSource, int taskCount)
131
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
129
132
  {
130
- assertThat(taskCount, is(0));
133
+ assertEquals(taskCount, 0);
131
134
  return emptyTaskReports(taskCount);
132
135
  }
133
136
  });
134
137
 
135
- Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
138
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
136
139
  method.setAccessible(true);
137
- Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
138
- List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
139
- assertThat(fileList.size(), is(0));
140
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
141
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
142
+ assertEquals(fileList.size(), 0);
140
143
  }
141
144
 
142
145
  @Test
143
146
  @SuppressWarnings("unchecked")
144
147
  public void testListFiles() throws Exception
145
148
  {
146
- List<String> expected = Arrays.asList(
149
+ final List<String> expected = Arrays.asList(
150
+ FTP_TEST_PATH_PREFIX + "01.csv",
151
+ FTP_TEST_PATH_PREFIX + "02.csv"
152
+ );
153
+
154
+ final ConfigSource config = config();
155
+ final PluginTask task = config.loadConfig(PluginTask.class);
156
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
157
+ @Override
158
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
159
+ {
160
+ assertEquals(taskCount, 2);
161
+ return emptyTaskReports(taskCount);
162
+ }
163
+ });
164
+
165
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
166
+ method.setAccessible(true);
167
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
168
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
169
+
170
+ assertEquals(fileList.get(0), expected.get(0));
171
+ assertEquals(fileList.get(1), expected.get(1));
172
+ assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
173
+ }
174
+
175
+ @Test
176
+ @SuppressWarnings("unchecked")
177
+ public void testListFileWithEmptyPattern() throws Exception
178
+ {
179
+ final List<String> expected = Arrays.asList(
147
180
  FTP_TEST_PATH_PREFIX + "01.csv",
148
181
  FTP_TEST_PATH_PREFIX + "02.csv"
149
182
  );
150
183
 
151
- ConfigSource config = config();
184
+ final ConfigSource config = config();
185
+ config.set("path_match_pattern", "");
186
+ final PluginTask task = config.loadConfig(PluginTask.class);
187
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
188
+ @Override
189
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
190
+ {
191
+ assertEquals(taskCount, 2);
192
+ return emptyTaskReports(taskCount);
193
+ }
194
+ });
195
+
196
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
197
+ method.setAccessible(true);
198
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
199
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
200
+
201
+ assertEquals(fileList.get(0), expected.get(0));
202
+ assertEquals(fileList.get(1), expected.get(1));
203
+ assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
204
+ }
205
+
206
+ @Test
207
+ @SuppressWarnings("unchecked")
208
+ public void testListFileWithSpacesPattern() throws Exception
209
+ {
210
+ final List<String> expected = Arrays.asList(
211
+ FTP_TEST_PATH_PREFIX + "01.csv",
212
+ FTP_TEST_PATH_PREFIX + "02.csv"
213
+ );
214
+
215
+ final ConfigSource config = config();
216
+ config.set("path_match_pattern", " ");
217
+ final PluginTask task = config.loadConfig(PluginTask.class);
218
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
219
+ @Override
220
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
221
+ {
222
+ assertEquals(taskCount, 2);
223
+ return emptyTaskReports(taskCount);
224
+ }
225
+ });
226
+
227
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
228
+ method.setAccessible(true);
229
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
230
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
231
+
232
+ assertEquals(fileList.get(0), expected.get(0));
233
+ assertEquals(fileList.get(1), expected.get(1));
234
+ assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
235
+ }
236
+
237
+ @Test
238
+ @SuppressWarnings("unchecked")
239
+ public void testListFilesWithPathMatcher() throws Exception
240
+ {
241
+ final String pattern = FTP_TEST_PATH_PREFIX + "02";
242
+ final Pattern pathMatchPattern = Pattern.compile(pattern);
243
+ final List<String> expected = Arrays.asList(
244
+ FTP_TEST_PATH_PREFIX + "02.csv"
245
+ );
246
+
247
+ final ConfigSource config = config();
248
+ config.set("path_match_pattern", pattern);
152
249
  final PluginTask task = config.loadConfig(PluginTask.class);
153
- ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
250
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
154
251
  @Override
155
- public List<TaskReport> run(TaskSource taskSource, int taskCount)
252
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
156
253
  {
157
- assertThat(taskCount, is(2));
254
+ assertEquals(taskCount, 1);
158
255
  return emptyTaskReports(taskCount);
159
256
  }
160
257
  });
161
258
 
162
- Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
259
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
163
260
  method.setAccessible(true);
164
- Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
165
- List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
166
- assertThat(fileList.get(0), is(expected.get(0)));
167
- assertThat(fileList.get(1), is(expected.get(1)));
168
- assertThat(configDiff.get(String.class, "last_path"), is(FTP_TEST_PATH_PREFIX + "02.csv"));
261
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
262
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
263
+
264
+ assertEquals(fileList.get(0), expected.get(0));
265
+ assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
266
+ }
267
+
268
+ @Test
269
+ @SuppressWarnings("unchecked")
270
+ public void testListFilesWithNonExistPathMatcher() throws Exception
271
+ {
272
+ final String pattern = "non_exist_file_matcher";
273
+ final Pattern pathMatchPattern = Pattern.compile(pattern);
274
+
275
+ final ConfigSource config = config();
276
+ config.set("path_match_pattern", pattern);
277
+ final PluginTask task = config.loadConfig(PluginTask.class);
278
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
279
+ @Override
280
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
281
+ {
282
+ assertEquals(taskCount, 0);
283
+ return emptyTaskReports(taskCount);
284
+ }
285
+ });
286
+
287
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
288
+ method.setAccessible(true);
289
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
290
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
291
+
292
+ assertEquals(fileList.size(), 0);
293
+ assertEquals(configDiff.get(String.class, "last_path"), "");
294
+ }
295
+
296
+ @Test
297
+ @SuppressWarnings("unchecked")
298
+ public void testListFilesWithNotExactPathMatcher() throws Exception
299
+ {
300
+ final String pattern = "\\b" + FTP_TEST_PATH_PREFIX + "\\b";
301
+ final Pattern pathMatchPattern = Pattern.compile(pattern);
302
+
303
+ final ConfigSource config = config();
304
+ config.set("path_match_pattern", pattern);
305
+ final PluginTask task = config.loadConfig(PluginTask.class);
306
+ final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
307
+ @Override
308
+ public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
309
+ {
310
+ assertEquals(taskCount, 0);
311
+ return emptyTaskReports(taskCount);
312
+ }
313
+ });
314
+
315
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
316
+ method.setAccessible(true);
317
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
318
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
319
+
320
+ assertEquals(fileList.size(), 0);
321
+ assertEquals(configDiff.get(String.class, "last_path"), "");
169
322
  }
170
323
 
171
324
  @Test
172
325
  public void testListFilesByPrefixIncrementalFalse()
173
326
  {
174
- ConfigSource config = config()
327
+ final ConfigSource config = config()
175
328
  .deepCopy()
176
329
  .set("incremental", false);
177
330
 
178
- ConfigDiff configDiff = runner.transaction(config, new Control());
331
+ final ConfigDiff configDiff = runner.transaction(config, new Control());
179
332
 
180
- assertThat(configDiff.toString(), is("{}"));
333
+ Assert.assertEquals(configDiff.toString(), "{}");
181
334
  }
182
335
 
183
336
  @Test
184
337
  @SuppressWarnings("unchecked")
185
338
  public void testFtpFileInputByOpen() throws Exception
186
339
  {
187
- ConfigSource config = config();
188
- PluginTask task = config().loadConfig(PluginTask.class);
340
+ final ConfigSource config = config();
341
+ final PluginTask task = config().loadConfig(PluginTask.class);
189
342
 
190
343
  runner.transaction(config, new Control());
191
344
 
192
- Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
345
+ final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
193
346
  method.setAccessible(true);
194
- Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
195
- List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
347
+ final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
348
+ final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
196
349
  task.setFiles(fileList);
197
350
 
198
351
  assertRecords(config, output);
199
352
  }
200
353
 
201
- private static List<TaskReport> emptyTaskReports(int taskCount)
354
+ private static List<TaskReport> emptyTaskReports(final int taskCount)
202
355
  {
203
- ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
356
+ final ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
204
357
  for (int i = 0; i < taskCount; i++) {
205
358
  reports.add(Exec.newTaskReport());
206
359
  }
@@ -211,9 +364,9 @@ public class TestFtpFileInputPlugin
211
364
  implements InputPlugin.Control
212
365
  {
213
366
  @Override
214
- public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
367
+ public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
215
368
  {
216
- List<TaskReport> reports = new ArrayList<>();
369
+ final List<TaskReport> reports = new ArrayList<>();
217
370
  for (int i = 0; i < taskCount; i++) {
218
371
  reports.add(runner.run(taskSource, schema, i, output));
219
372
  }
@@ -239,9 +392,9 @@ public class TestFtpFileInputPlugin
239
392
  .set("parser", parserConfig(schemaConfig()));
240
393
  }
241
394
 
242
- private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
395
+ private ImmutableMap<String, Object> parserConfig(final ImmutableList<Object> schemaConfig)
243
396
  {
244
- ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
397
+ final ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
245
398
  builder.put("type", "csv");
246
399
  builder.put("newline", "CRLF");
247
400
  builder.put("delimiter", ",");
@@ -257,7 +410,7 @@ public class TestFtpFileInputPlugin
257
410
 
258
411
  private ImmutableList<Object> schemaConfig()
259
412
  {
260
- ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
413
+ final ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
261
414
  builder.add(ImmutableMap.of("name", "id", "type", "long"));
262
415
  builder.add(ImmutableMap.of("name", "account", "type", "long"));
263
416
  builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
@@ -266,37 +419,37 @@ public class TestFtpFileInputPlugin
266
419
  return builder.build();
267
420
  }
268
421
 
269
- public SSLPlugins.SSLPluginConfig sslConfig(PluginTask task)
422
+ public SSLPlugins.SSLPluginConfig sslConfig(final PluginTask task)
270
423
  {
271
424
  return SSLPlugins.configure(task);
272
425
  }
273
426
 
274
- private void assertRecords(ConfigSource config, MockPageOutput output)
427
+ private void assertRecords(final ConfigSource config, final MockPageOutput output)
275
428
  {
276
- List<Object[]> records = getRecords(config, output);
277
- assertThat(records.size(), is(8));
429
+ final List<Object[]> records = getRecords(config, output);
430
+ assertEquals(records.size(), 8);
278
431
  {
279
- Object[] record = records.get(0);
280
- assertThat((long) record[0], is(1L));
281
- assertThat((long) record[1], is(32864L));
282
- assertThat(record[2].toString(), is("2015-01-27 19:23:49 UTC"));
283
- assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
284
- assertThat(record[4].toString(), is("embulk"));
432
+ final Object[] record = records.get(0);
433
+ assertEquals((long) record[0], 1L);
434
+ assertEquals((long) record[1], 32864L);
435
+ assertEquals(record[2].toString(), "2015-01-27 19:23:49 UTC");
436
+ assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
437
+ assertEquals(record[4].toString(), "embulk");
285
438
  }
286
439
 
287
440
  {
288
- Object[] record = records.get(1);
289
- assertThat((long) record[0], is(2L));
290
- assertThat((long) record[1], is(14824L));
291
- assertThat(record[2].toString(), is("2015-01-27 19:01:23 UTC"));
292
- assertThat(record[3].toString(), is("2015-01-27 00:00:00 UTC"));
293
- assertThat(record[4].toString(), is("embulk jruby"));
441
+ final Object[] record = records.get(1);
442
+ assertEquals((long) record[0], 2L);
443
+ assertEquals((long) record[1], 14824L);
444
+ assertEquals(record[2].toString(), "2015-01-27 19:01:23 UTC");
445
+ assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
446
+ assertEquals(record[4].toString(), "embulk jruby");
294
447
  }
295
448
  }
296
449
 
297
- private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
450
+ private List<Object[]> getRecords(final ConfigSource config, final MockPageOutput output)
298
451
  {
299
- Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
452
+ final Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
300
453
  return Pages.toObjects(schema, output.pages);
301
454
  }
302
455
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-input-ftp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sadayuki Furuhashi
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-12-26 00:00:00.000000000 Z
11
+ date: 2019-11-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -52,10 +52,10 @@ files:
52
52
  - src/test/resources/sample_01.csv
53
53
  - src/test/resources/sample_02.csv
54
54
  - classpath/bcpkix-jdk15on-1.52.jar
55
- - classpath/embulk-util-ftp-0.2.0.jar
55
+ - classpath/embulk-util-ftp-0.2.1.jar
56
56
  - classpath/bcprov-jdk15on-1.52.jar
57
57
  - classpath/ftp4j-1.7.2.jar
58
- - classpath/embulk-input-ftp-0.2.0.jar
58
+ - classpath/embulk-input-ftp-0.2.1.jar
59
59
  homepage: https://github.com/embulk/embulk-input-ftp
60
60
  licenses:
61
61
  - Apache 2.0