embulk-input-ftp 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/classpath/embulk-input-ftp-0.2.1.jar +0 -0
- data/classpath/{embulk-util-ftp-0.2.0.jar → embulk-util-ftp-0.2.1.jar} +0 -0
- data/src/main/java/org/embulk/input/FtpFileInputPlugin.java +97 -69
- data/src/test/java/org/embulk/input/TestFtpFileInputPlugin.java +213 -60
- metadata +4 -4
- data/classpath/embulk-input-ftp-0.2.0.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 803a0b43863b8ec7aedb1de35984911526561704
|
4
|
+
data.tar.gz: decc88b75fa71e334b64ff8853c00be0ea1c8bcf
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 04e1fa0d6a4f8e3c46e149a22fe6e621b7e7dbfab297a0fb2f2730738bf9395e75d628f7a7e247d3a3f4caac4cf66d4dacbc6cacd129a1fbb1a850ea4c73f557
|
7
|
+
data.tar.gz: 436aba43e3f26efdbd3d1fbd079e6c9ffb183ea881c4048bbc87196754ac9e241703e011cd1e01fd881d85a76e69afb34d12f2264a7ecc98a17831c49196530e
|
Binary file
|
Binary file
|
@@ -14,6 +14,8 @@ import it.sauronsoftware.ftp4j.FTPException;
|
|
14
14
|
import it.sauronsoftware.ftp4j.FTPFile;
|
15
15
|
import it.sauronsoftware.ftp4j.FTPIllegalReplyException;
|
16
16
|
import it.sauronsoftware.ftp4j.FTPListParseException;
|
17
|
+
|
18
|
+
import org.apache.commons.lang3.StringUtils;
|
17
19
|
import org.embulk.config.Config;
|
18
20
|
import org.embulk.config.ConfigDefault;
|
19
21
|
import org.embulk.config.ConfigDiff;
|
@@ -46,6 +48,7 @@ import java.util.List;
|
|
46
48
|
import java.util.Optional;
|
47
49
|
import java.util.concurrent.ExecutorService;
|
48
50
|
import java.util.concurrent.Executors;
|
51
|
+
import java.util.regex.Pattern;
|
49
52
|
|
50
53
|
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
51
54
|
|
@@ -67,6 +70,10 @@ public class FtpFileInputPlugin
|
|
67
70
|
@ConfigDefault("null")
|
68
71
|
Optional<String> getLastPath();
|
69
72
|
|
73
|
+
@Config("path_match_pattern")
|
74
|
+
@ConfigDefault("\".*\"")
|
75
|
+
String getPathMatchPattern();
|
76
|
+
|
70
77
|
@Config("incremental")
|
71
78
|
@ConfigDefault("true")
|
72
79
|
boolean getIncremental();
|
@@ -113,14 +120,22 @@ public class FtpFileInputPlugin
|
|
113
120
|
}
|
114
121
|
|
115
122
|
@Override
|
116
|
-
public ConfigDiff transaction(ConfigSource config, FileInputPlugin.Control control)
|
123
|
+
public ConfigDiff transaction(final ConfigSource config, final FileInputPlugin.Control control)
|
117
124
|
{
|
118
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
125
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
119
126
|
|
120
127
|
task.setSSLConfig(SSLPlugins.configure(task));
|
121
128
|
|
129
|
+
String pattern = task.getPathMatchPattern();
|
130
|
+
// If pattern is empty then use default pattern
|
131
|
+
if (StringUtils.trim(pattern).isEmpty()) {
|
132
|
+
pattern = ".*";
|
133
|
+
}
|
134
|
+
// Create path match pattern
|
135
|
+
final Pattern pathMatchPattern = Pattern.compile(pattern);
|
136
|
+
|
122
137
|
// list files recursively
|
123
|
-
List<String> files = listFiles(log, task);
|
138
|
+
final List<String> files = listFiles(log, task, pathMatchPattern);
|
124
139
|
task.setFiles(files);
|
125
140
|
log.info("Using files {}", files);
|
126
141
|
|
@@ -131,16 +146,16 @@ public class FtpFileInputPlugin
|
|
131
146
|
}
|
132
147
|
|
133
148
|
@Override
|
134
|
-
public ConfigDiff resume(TaskSource taskSource,
|
135
|
-
int taskCount,
|
136
|
-
FileInputPlugin.Control control)
|
149
|
+
public ConfigDiff resume(final TaskSource taskSource,
|
150
|
+
final int taskCount,
|
151
|
+
final FileInputPlugin.Control control)
|
137
152
|
{
|
138
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
153
|
+
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
139
154
|
|
140
155
|
control.run(taskSource, taskCount);
|
141
156
|
|
142
157
|
// build next config
|
143
|
-
ConfigDiff configDiff = Exec.newConfigDiff();
|
158
|
+
final ConfigDiff configDiff = Exec.newConfigDiff();
|
144
159
|
|
145
160
|
// last_path
|
146
161
|
if (task.getIncremental()) {
|
@@ -151,7 +166,7 @@ public class FtpFileInputPlugin
|
|
151
166
|
}
|
152
167
|
}
|
153
168
|
else {
|
154
|
-
List<String> files = new ArrayList<String>(task.getFiles());
|
169
|
+
final List<String> files = new ArrayList<String>(task.getFiles());
|
155
170
|
Collections.sort(files);
|
156
171
|
configDiff.set("last_path", files.get(files.size() - 1));
|
157
172
|
}
|
@@ -161,14 +176,14 @@ public class FtpFileInputPlugin
|
|
161
176
|
}
|
162
177
|
|
163
178
|
@Override
|
164
|
-
public void cleanup(TaskSource taskSource,
|
165
|
-
int taskCount,
|
166
|
-
List<TaskReport> successTaskReports)
|
179
|
+
public void cleanup(final TaskSource taskSource,
|
180
|
+
final int taskCount,
|
181
|
+
final List<TaskReport> successTaskReports)
|
167
182
|
{
|
168
183
|
// do nothing
|
169
184
|
}
|
170
185
|
|
171
|
-
private static FTPClient newFTPClient(Logger log, PluginTask task)
|
186
|
+
private static FTPClient newFTPClient(final Logger log, final PluginTask task)
|
172
187
|
{
|
173
188
|
FTPClient client = new FTPClient();
|
174
189
|
try {
|
@@ -186,14 +201,14 @@ public class FtpFileInputPlugin
|
|
186
201
|
log.info("Using FTPS(FTPS/implicit) mode");
|
187
202
|
}
|
188
203
|
}
|
189
|
-
int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
|
204
|
+
final int port = task.getPort().isPresent() ? task.getPort().get() : defaultPort;
|
190
205
|
|
191
206
|
client.addCommunicationListener(new LoggingCommunicationListner(log));
|
192
207
|
|
193
208
|
// TODO configurable timeout parameters
|
194
209
|
client.setAutoNoopTimeout(3000);
|
195
210
|
|
196
|
-
FTPConnector con = client.getConnector();
|
211
|
+
final FTPConnector con = client.getConnector();
|
197
212
|
con.setConnectionTimeout(30);
|
198
213
|
con.setReadTimeout(60);
|
199
214
|
con.setCloseTimeout(60);
|
@@ -230,19 +245,19 @@ public class FtpFileInputPlugin
|
|
230
245
|
client.setCompressionEnabled(true);
|
231
246
|
}
|
232
247
|
|
233
|
-
FTPClient connected = client;
|
248
|
+
final FTPClient connected = client;
|
234
249
|
client = null;
|
235
250
|
return connected;
|
236
251
|
}
|
237
|
-
catch (FTPException ex) {
|
252
|
+
catch (final FTPException ex) {
|
238
253
|
log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
|
239
254
|
throw Throwables.propagate(ex);
|
240
255
|
}
|
241
|
-
catch (FTPIllegalReplyException ex) {
|
256
|
+
catch (final FTPIllegalReplyException ex) {
|
242
257
|
log.info("FTP protocol error");
|
243
258
|
throw Throwables.propagate(ex);
|
244
259
|
}
|
245
|
-
catch (IOException ex) {
|
260
|
+
catch (final IOException ex) {
|
246
261
|
log.info("FTP network error: " + ex);
|
247
262
|
throw Throwables.propagate(ex);
|
248
263
|
}
|
@@ -253,37 +268,37 @@ public class FtpFileInputPlugin
|
|
253
268
|
}
|
254
269
|
}
|
255
270
|
|
256
|
-
static void disconnectClient(FTPClient client)
|
271
|
+
static void disconnectClient(final FTPClient client)
|
257
272
|
{
|
258
273
|
if (client.isConnected()) {
|
259
274
|
try {
|
260
275
|
client.disconnect(false);
|
261
276
|
}
|
262
|
-
catch (FTPException ex) {
|
277
|
+
catch (final FTPException ex) {
|
263
278
|
// do nothing
|
264
279
|
}
|
265
|
-
catch (FTPIllegalReplyException ex) {
|
280
|
+
catch (final FTPIllegalReplyException ex) {
|
266
281
|
// do nothing
|
267
282
|
}
|
268
|
-
catch (IOException ex) {
|
283
|
+
catch (final IOException ex) {
|
269
284
|
// do nothing
|
270
285
|
}
|
271
286
|
}
|
272
287
|
}
|
273
288
|
|
274
|
-
private List<String> listFiles(Logger log, PluginTask task)
|
289
|
+
private List<String> listFiles(final Logger log, final PluginTask task, final Pattern pathMatchPattern)
|
275
290
|
{
|
276
|
-
FTPClient client = newFTPClient(log, task);
|
291
|
+
final FTPClient client = newFTPClient(log, task);
|
277
292
|
try {
|
278
|
-
return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath());
|
293
|
+
return listFilesByPrefix(log, client, task.getPathPrefix(), task.getLastPath(), pathMatchPattern);
|
279
294
|
}
|
280
295
|
finally {
|
281
296
|
disconnectClient(client);
|
282
297
|
}
|
283
298
|
}
|
284
299
|
|
285
|
-
public static List<String> listFilesByPrefix(Logger log, FTPClient client,
|
286
|
-
String prefix, Optional<String> lastPath)
|
300
|
+
public static List<String> listFilesByPrefix(final Logger log, final FTPClient client,
|
301
|
+
final String prefix, final Optional<String> lastPath, final Pattern pathMatchPattern)
|
287
302
|
{
|
288
303
|
String directory;
|
289
304
|
String fileNamePrefix;
|
@@ -292,7 +307,7 @@ public class FtpFileInputPlugin
|
|
292
307
|
fileNamePrefix = "";
|
293
308
|
}
|
294
309
|
else {
|
295
|
-
int pos = prefix.lastIndexOf("/");
|
310
|
+
final int pos = prefix.lastIndexOf("/");
|
296
311
|
if (pos < 0) {
|
297
312
|
directory = "";
|
298
313
|
fileNamePrefix = prefix;
|
@@ -303,7 +318,7 @@ public class FtpFileInputPlugin
|
|
303
318
|
}
|
304
319
|
}
|
305
320
|
|
306
|
-
ImmutableList.Builder<String> builder = ImmutableList.builder();
|
321
|
+
final ImmutableList.Builder<String> builder = ImmutableList.builder();
|
307
322
|
|
308
323
|
try {
|
309
324
|
String currentDirectory = client.currentDirectory();
|
@@ -314,33 +329,33 @@ public class FtpFileInputPlugin
|
|
314
329
|
currentDirectory = directory;
|
315
330
|
}
|
316
331
|
|
317
|
-
for (FTPFile file : client.list()) {
|
332
|
+
for (final FTPFile file : client.list()) {
|
318
333
|
if (file.getName().startsWith(fileNamePrefix)) {
|
319
|
-
listFilesRecursive(client, currentDirectory, file, lastPath, builder);
|
334
|
+
listFilesRecursive(client, currentDirectory, file, lastPath, builder, pathMatchPattern);
|
320
335
|
}
|
321
336
|
}
|
322
337
|
}
|
323
|
-
catch (FTPListParseException ex) {
|
338
|
+
catch (final FTPListParseException ex) {
|
324
339
|
log.info("FTP listing files failed");
|
325
340
|
throw Throwables.propagate(ex);
|
326
341
|
}
|
327
|
-
catch (FTPAbortedException ex) {
|
342
|
+
catch (final FTPAbortedException ex) {
|
328
343
|
log.info("FTP listing files failed");
|
329
344
|
throw Throwables.propagate(ex);
|
330
345
|
}
|
331
|
-
catch (FTPDataTransferException ex) {
|
346
|
+
catch (final FTPDataTransferException ex) {
|
332
347
|
log.info("FTP data transfer failed");
|
333
348
|
throw Throwables.propagate(ex);
|
334
349
|
}
|
335
|
-
catch (FTPException ex) {
|
350
|
+
catch (final FTPException ex) {
|
336
351
|
log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
|
337
352
|
throw Throwables.propagate(ex);
|
338
353
|
}
|
339
|
-
catch (FTPIllegalReplyException ex) {
|
354
|
+
catch (final FTPIllegalReplyException ex) {
|
340
355
|
log.info("FTP protocol error");
|
341
356
|
throw Throwables.propagate(ex);
|
342
357
|
}
|
343
|
-
catch (IOException ex) {
|
358
|
+
catch (final IOException ex) {
|
344
359
|
log.info("FTP network error: " + ex);
|
345
360
|
throw Throwables.propagate(ex);
|
346
361
|
}
|
@@ -348,15 +363,15 @@ public class FtpFileInputPlugin
|
|
348
363
|
return builder.build();
|
349
364
|
}
|
350
365
|
|
351
|
-
private static void listFilesRecursive(FTPClient client,
|
352
|
-
String baseDirectoryPath, FTPFile file, Optional<String> lastPath,
|
353
|
-
ImmutableList.Builder<String> builder)
|
366
|
+
private static void listFilesRecursive(final FTPClient client,
|
367
|
+
String baseDirectoryPath, final FTPFile file, final Optional<String> lastPath,
|
368
|
+
final ImmutableList.Builder<String> builder, final Pattern pathMatchPattern)
|
354
369
|
throws IOException, FTPException, FTPIllegalReplyException, FTPDataTransferException, FTPAbortedException, FTPListParseException
|
355
370
|
{
|
356
371
|
if (!baseDirectoryPath.endsWith("/")) {
|
357
372
|
baseDirectoryPath = baseDirectoryPath + "/";
|
358
373
|
}
|
359
|
-
String path = baseDirectoryPath + file.getName();
|
374
|
+
final String path = baseDirectoryPath + file.getName();
|
360
375
|
|
361
376
|
if (lastPath.isPresent() && path.compareTo(lastPath.get()) <= 0) {
|
362
377
|
return;
|
@@ -364,12 +379,14 @@ public class FtpFileInputPlugin
|
|
364
379
|
|
365
380
|
switch (file.getType()) {
|
366
381
|
case FTPFile.TYPE_FILE:
|
367
|
-
|
382
|
+
if (pathMatchPattern.matcher(path).find()) {
|
383
|
+
builder.add(path);
|
384
|
+
}
|
368
385
|
break;
|
369
386
|
case FTPFile.TYPE_DIRECTORY:
|
370
387
|
client.changeDirectory(path);
|
371
|
-
for (FTPFile subFile : client.list()) {
|
372
|
-
listFilesRecursive(client, path, subFile, lastPath, builder);
|
388
|
+
for (final FTPFile subFile : client.list()) {
|
389
|
+
listFilesRecursive(client, path, subFile, lastPath, builder, pathMatchPattern);
|
373
390
|
}
|
374
391
|
client.changeDirectory(baseDirectoryPath);
|
375
392
|
break;
|
@@ -379,9 +396,9 @@ public class FtpFileInputPlugin
|
|
379
396
|
}
|
380
397
|
|
381
398
|
@Override
|
382
|
-
public TransactionalFileInput open(TaskSource taskSource, int taskIndex)
|
399
|
+
public TransactionalFileInput open(final TaskSource taskSource, final int taskIndex)
|
383
400
|
{
|
384
|
-
PluginTask task = taskSource.loadTask(PluginTask.class);
|
401
|
+
final PluginTask task = taskSource.loadTask(PluginTask.class);
|
385
402
|
return new FtpFileInput(log, task, taskIndex);
|
386
403
|
}
|
387
404
|
|
@@ -390,17 +407,19 @@ public class FtpFileInputPlugin
|
|
390
407
|
{
|
391
408
|
private final Logger log;
|
392
409
|
|
393
|
-
public LoggingCommunicationListner(Logger log)
|
410
|
+
public LoggingCommunicationListner(final Logger log)
|
394
411
|
{
|
395
412
|
this.log = log;
|
396
413
|
}
|
397
414
|
|
398
|
-
|
415
|
+
@Override
|
416
|
+
public void received(final String statement)
|
399
417
|
{
|
400
418
|
log.info("< " + statement);
|
401
419
|
}
|
402
420
|
|
403
|
-
|
421
|
+
@Override
|
422
|
+
public void sent(final String statement)
|
404
423
|
{
|
405
424
|
if (statement.startsWith("PASS")) {
|
406
425
|
// don't show password
|
@@ -419,19 +438,21 @@ public class FtpFileInputPlugin
|
|
419
438
|
private long totalTransfer;
|
420
439
|
private long nextTransferNotice;
|
421
440
|
|
422
|
-
public LoggingTransferListener(Logger log, long transferNoticeBytes)
|
441
|
+
public LoggingTransferListener(final Logger log, final long transferNoticeBytes)
|
423
442
|
{
|
424
443
|
this.log = log;
|
425
444
|
this.transferNoticeBytes = transferNoticeBytes;
|
426
445
|
this.nextTransferNotice = transferNoticeBytes;
|
427
446
|
}
|
428
447
|
|
448
|
+
@Override
|
429
449
|
public void started()
|
430
450
|
{
|
431
451
|
log.info("Transfer started");
|
432
452
|
}
|
433
453
|
|
434
|
-
|
454
|
+
@Override
|
455
|
+
public void transferred(final int length)
|
435
456
|
{
|
436
457
|
totalTransfer += length;
|
437
458
|
if (totalTransfer > nextTransferNotice) {
|
@@ -440,16 +461,19 @@ public class FtpFileInputPlugin
|
|
440
461
|
}
|
441
462
|
}
|
442
463
|
|
464
|
+
@Override
|
443
465
|
public void completed()
|
444
466
|
{
|
445
467
|
log.info("Transfer completed " + totalTransfer + " bytes");
|
446
468
|
}
|
447
469
|
|
470
|
+
@Override
|
448
471
|
public void aborted()
|
449
472
|
{
|
450
473
|
log.info("Transfer aborted");
|
451
474
|
}
|
452
475
|
|
476
|
+
@Override
|
453
477
|
public void failed()
|
454
478
|
{
|
455
479
|
log.info("Transfer failed");
|
@@ -459,43 +483,45 @@ public class FtpFileInputPlugin
|
|
459
483
|
private static final long TRANSFER_NOTICE_BYTES = 100 * 1024 * 1024;
|
460
484
|
|
461
485
|
private static InputStream startDownload(final Logger log, final FTPClient client,
|
462
|
-
final String path, final long offset, ExecutorService executor)
|
486
|
+
final String path, final long offset, final ExecutorService executor)
|
463
487
|
{
|
464
|
-
BlockingTransfer t = BlockingTransfer.submit(executor,
|
488
|
+
final BlockingTransfer t = BlockingTransfer.submit(executor,
|
465
489
|
new Function<BlockingTransfer, Runnable>()
|
466
490
|
{
|
491
|
+
@Override
|
467
492
|
public Runnable apply(final BlockingTransfer transfer)
|
468
493
|
{
|
469
494
|
return new Runnable() {
|
495
|
+
@Override
|
470
496
|
public void run()
|
471
497
|
{
|
472
498
|
try {
|
473
499
|
client.download(path, Channels.newOutputStream(transfer.getWriterChannel()), offset, new LoggingTransferListener(log, TRANSFER_NOTICE_BYTES));
|
474
500
|
}
|
475
|
-
catch (FTPException ex) {
|
501
|
+
catch (final FTPException ex) {
|
476
502
|
log.info("FTP command failed: " + ex.getCode() + " " + ex.getMessage());
|
477
503
|
throw Throwables.propagate(ex);
|
478
504
|
}
|
479
|
-
catch (FTPDataTransferException ex) {
|
505
|
+
catch (final FTPDataTransferException ex) {
|
480
506
|
log.info("FTP data transfer failed");
|
481
507
|
throw Throwables.propagate(ex);
|
482
508
|
}
|
483
|
-
catch (FTPAbortedException ex) {
|
509
|
+
catch (final FTPAbortedException ex) {
|
484
510
|
log.info("FTP listing files failed");
|
485
511
|
throw Throwables.propagate(ex);
|
486
512
|
}
|
487
|
-
catch (FTPIllegalReplyException ex) {
|
513
|
+
catch (final FTPIllegalReplyException ex) {
|
488
514
|
log.info("FTP protocol error");
|
489
515
|
throw Throwables.propagate(ex);
|
490
516
|
}
|
491
|
-
catch (IOException ex) {
|
517
|
+
catch (final IOException ex) {
|
492
518
|
throw Throwables.propagate(ex);
|
493
519
|
}
|
494
520
|
finally {
|
495
521
|
try {
|
496
522
|
transfer.getWriterChannel().close();
|
497
523
|
}
|
498
|
-
catch (IOException ex) {
|
524
|
+
catch (final IOException ex) {
|
499
525
|
throw new RuntimeException(ex);
|
500
526
|
}
|
501
527
|
}
|
@@ -514,7 +540,7 @@ public class FtpFileInputPlugin
|
|
514
540
|
private final ExecutorService executor;
|
515
541
|
private final String path;
|
516
542
|
|
517
|
-
public FtpInputStreamReopener(Logger log, FTPClient client, ExecutorService executor, String path)
|
543
|
+
public FtpInputStreamReopener(final Logger log, final FTPClient client, final ExecutorService executor, final String path)
|
518
544
|
{
|
519
545
|
this.log = log;
|
520
546
|
this.client = client;
|
@@ -539,16 +565,16 @@ public class FtpFileInputPlugin
|
|
539
565
|
}
|
540
566
|
|
541
567
|
@Override
|
542
|
-
public boolean isRetryableException(Exception exception)
|
568
|
+
public boolean isRetryableException(final Exception exception)
|
543
569
|
{
|
544
570
|
return true; // TODO
|
545
571
|
}
|
546
572
|
|
547
573
|
@Override
|
548
|
-
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
574
|
+
public void onRetry(final Exception exception, final int retryCount, final int retryLimit, final int retryWait)
|
549
575
|
throws RetryGiveupException
|
550
576
|
{
|
551
|
-
String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
|
577
|
+
final String message = String.format("FTP GET request failed. Retrying %d/%d after %d seconds. Message: %s",
|
552
578
|
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
553
579
|
if (retryCount % 3 == 0) {
|
554
580
|
log.warn(message, exception);
|
@@ -559,17 +585,17 @@ public class FtpFileInputPlugin
|
|
559
585
|
}
|
560
586
|
|
561
587
|
@Override
|
562
|
-
public void onGiveup(Exception firstException, Exception lastException)
|
588
|
+
public void onGiveup(final Exception firstException, final Exception lastException)
|
563
589
|
throws RetryGiveupException
|
564
590
|
{
|
565
591
|
}
|
566
592
|
});
|
567
593
|
}
|
568
|
-
catch (RetryGiveupException ex) {
|
594
|
+
catch (final RetryGiveupException ex) {
|
569
595
|
Throwables.propagateIfInstanceOf(ex.getCause(), IOException.class);
|
570
596
|
throw Throwables.propagate(ex.getCause());
|
571
597
|
}
|
572
|
-
catch (InterruptedException ex) {
|
598
|
+
catch (final InterruptedException ex) {
|
573
599
|
throw new InterruptedIOException();
|
574
600
|
}
|
575
601
|
}
|
@@ -585,7 +611,7 @@ public class FtpFileInputPlugin
|
|
585
611
|
private final String path;
|
586
612
|
private boolean opened = false;
|
587
613
|
|
588
|
-
public SingleFileProvider(Logger log, PluginTask task, int taskIndex)
|
614
|
+
public SingleFileProvider(final Logger log, final PluginTask task, final int taskIndex)
|
589
615
|
{
|
590
616
|
this.log = log;
|
591
617
|
this.client = newFTPClient(log, task);
|
@@ -628,15 +654,17 @@ public class FtpFileInputPlugin
|
|
628
654
|
extends InputStreamFileInput
|
629
655
|
implements TransactionalFileInput
|
630
656
|
{
|
631
|
-
public FtpFileInput(Logger log, PluginTask task, int taskIndex)
|
657
|
+
public FtpFileInput(final Logger log, final PluginTask task, final int taskIndex)
|
632
658
|
{
|
633
659
|
super(task.getBufferAllocator(), new SingleFileProvider(log, task, taskIndex));
|
634
660
|
}
|
635
661
|
|
662
|
+
@Override
|
636
663
|
public void abort()
|
637
664
|
{
|
638
665
|
}
|
639
666
|
|
667
|
+
@Override
|
640
668
|
public TaskReport commit()
|
641
669
|
{
|
642
670
|
return Exec.newTaskReport();
|
@@ -19,6 +19,7 @@ import org.embulk.spi.TestPageBuilderReader.MockPageOutput;
|
|
19
19
|
import org.embulk.spi.util.Pages;
|
20
20
|
import org.embulk.standards.CsvParserPlugin;
|
21
21
|
import org.embulk.util.ftp.SSLPlugins;
|
22
|
+
import org.junit.Assert;
|
22
23
|
import org.junit.Before;
|
23
24
|
import org.junit.BeforeClass;
|
24
25
|
import org.junit.Rule;
|
@@ -30,9 +31,9 @@ import java.util.ArrayList;
|
|
30
31
|
import java.util.Arrays;
|
31
32
|
import java.util.List;
|
32
33
|
import java.util.Map;
|
34
|
+
import java.util.regex.Pattern;
|
33
35
|
|
34
|
-
import static org.
|
35
|
-
import static org.junit.Assert.assertThat;
|
36
|
+
import static org.junit.Assert.assertEquals;
|
36
37
|
|
37
38
|
public class TestFtpFileInputPlugin
|
38
39
|
{
|
@@ -45,8 +46,10 @@ public class TestFtpFileInputPlugin
|
|
45
46
|
private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
|
46
47
|
private static String FTP_TEST_DIRECTORY;
|
47
48
|
private static String FTP_TEST_PATH_PREFIX;
|
49
|
+
|
48
50
|
private FileInputRunner runner;
|
49
51
|
private TestPageBuilderReader.MockPageOutput output;
|
52
|
+
private final Pattern defaultPathMatchPattern = Pattern.compile(".*");
|
50
53
|
|
51
54
|
/*
|
52
55
|
* This test case requires environment variables
|
@@ -86,7 +89,7 @@ public class TestFtpFileInputPlugin
|
|
86
89
|
@Test(expected = RuntimeException.class) // TODO ConfigException should be thrown
|
87
90
|
public void testTransactionWithInvalidHost()
|
88
91
|
{
|
89
|
-
ConfigSource config = config().deepCopy()
|
92
|
+
final ConfigSource config = config().deepCopy()
|
90
93
|
.set("host", "non-exists.example.com");
|
91
94
|
|
92
95
|
runner.transaction(config, new Control());
|
@@ -95,24 +98,24 @@ public class TestFtpFileInputPlugin
|
|
95
98
|
@Test
|
96
99
|
public void testResume()
|
97
100
|
{
|
98
|
-
PluginTask task = config().loadConfig(PluginTask.class);
|
101
|
+
final PluginTask task = config().loadConfig(PluginTask.class);
|
99
102
|
task.setSSLConfig(sslConfig(task));
|
100
103
|
task.setFiles(Arrays.asList("in/aa/a"));
|
101
|
-
ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
|
104
|
+
final ConfigDiff configDiff = plugin.resume(task.dump(), 0, new FileInputPlugin.Control()
|
102
105
|
{
|
103
106
|
@Override
|
104
|
-
public List<TaskReport> run(TaskSource taskSource, int taskCount)
|
107
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
105
108
|
{
|
106
109
|
return emptyTaskReports(taskCount);
|
107
110
|
}
|
108
111
|
});
|
109
|
-
|
112
|
+
assertEquals(configDiff.get(String.class, "last_path"), "in/aa/a");
|
110
113
|
}
|
111
114
|
|
112
115
|
@Test
|
113
116
|
public void testCleanup()
|
114
117
|
{
|
115
|
-
PluginTask task = config().loadConfig(PluginTask.class);
|
118
|
+
final PluginTask task = config().loadConfig(PluginTask.class);
|
116
119
|
plugin.cleanup(task.dump(), 0, Lists.<TaskReport>newArrayList()); // no errors happens
|
117
120
|
}
|
118
121
|
|
@@ -120,87 +123,237 @@ public class TestFtpFileInputPlugin
|
|
120
123
|
@SuppressWarnings("unchecked")
|
121
124
|
public void testListFilesWithNonExistPath() throws Exception
|
122
125
|
{
|
123
|
-
ConfigSource config = config().deepCopy()
|
126
|
+
final ConfigSource config = config().deepCopy()
|
124
127
|
.set("path_prefix", "non-exists-path");
|
125
|
-
PluginTask task = config.loadConfig(PluginTask.class);
|
128
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
126
129
|
plugin.transaction(config, new FileInputPlugin.Control() {
|
127
130
|
@Override
|
128
|
-
public List<TaskReport> run(TaskSource taskSource, int taskCount)
|
131
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
129
132
|
{
|
130
|
-
|
133
|
+
assertEquals(taskCount, 0);
|
131
134
|
return emptyTaskReports(taskCount);
|
132
135
|
}
|
133
136
|
});
|
134
137
|
|
135
|
-
Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
|
138
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
136
139
|
method.setAccessible(true);
|
137
|
-
Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
138
|
-
List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
|
139
|
-
|
140
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
141
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
|
142
|
+
assertEquals(fileList.size(), 0);
|
140
143
|
}
|
141
144
|
|
142
145
|
@Test
|
143
146
|
@SuppressWarnings("unchecked")
|
144
147
|
public void testListFiles() throws Exception
|
145
148
|
{
|
146
|
-
List<String> expected = Arrays.asList(
|
149
|
+
final List<String> expected = Arrays.asList(
|
150
|
+
FTP_TEST_PATH_PREFIX + "01.csv",
|
151
|
+
FTP_TEST_PATH_PREFIX + "02.csv"
|
152
|
+
);
|
153
|
+
|
154
|
+
final ConfigSource config = config();
|
155
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
156
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
157
|
+
@Override
|
158
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
159
|
+
{
|
160
|
+
assertEquals(taskCount, 2);
|
161
|
+
return emptyTaskReports(taskCount);
|
162
|
+
}
|
163
|
+
});
|
164
|
+
|
165
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
166
|
+
method.setAccessible(true);
|
167
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
168
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
|
169
|
+
|
170
|
+
assertEquals(fileList.get(0), expected.get(0));
|
171
|
+
assertEquals(fileList.get(1), expected.get(1));
|
172
|
+
assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
|
173
|
+
}
|
174
|
+
|
175
|
+
@Test
|
176
|
+
@SuppressWarnings("unchecked")
|
177
|
+
public void testListFileWithEmptyPattern() throws Exception
|
178
|
+
{
|
179
|
+
final List<String> expected = Arrays.asList(
|
147
180
|
FTP_TEST_PATH_PREFIX + "01.csv",
|
148
181
|
FTP_TEST_PATH_PREFIX + "02.csv"
|
149
182
|
);
|
150
183
|
|
151
|
-
ConfigSource config = config();
|
184
|
+
final ConfigSource config = config();
|
185
|
+
config.set("path_match_pattern", "");
|
186
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
187
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
188
|
+
@Override
|
189
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
190
|
+
{
|
191
|
+
assertEquals(taskCount, 2);
|
192
|
+
return emptyTaskReports(taskCount);
|
193
|
+
}
|
194
|
+
});
|
195
|
+
|
196
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
197
|
+
method.setAccessible(true);
|
198
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
199
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
|
200
|
+
|
201
|
+
assertEquals(fileList.get(0), expected.get(0));
|
202
|
+
assertEquals(fileList.get(1), expected.get(1));
|
203
|
+
assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
|
204
|
+
}
|
205
|
+
|
206
|
+
@Test
|
207
|
+
@SuppressWarnings("unchecked")
|
208
|
+
public void testListFileWithSpacesPattern() throws Exception
|
209
|
+
{
|
210
|
+
final List<String> expected = Arrays.asList(
|
211
|
+
FTP_TEST_PATH_PREFIX + "01.csv",
|
212
|
+
FTP_TEST_PATH_PREFIX + "02.csv"
|
213
|
+
);
|
214
|
+
|
215
|
+
final ConfigSource config = config();
|
216
|
+
config.set("path_match_pattern", " ");
|
217
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
218
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
219
|
+
@Override
|
220
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
221
|
+
{
|
222
|
+
assertEquals(taskCount, 2);
|
223
|
+
return emptyTaskReports(taskCount);
|
224
|
+
}
|
225
|
+
});
|
226
|
+
|
227
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
228
|
+
method.setAccessible(true);
|
229
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
230
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
|
231
|
+
|
232
|
+
assertEquals(fileList.get(0), expected.get(0));
|
233
|
+
assertEquals(fileList.get(1), expected.get(1));
|
234
|
+
assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
|
235
|
+
}
|
236
|
+
|
237
|
+
@Test
|
238
|
+
@SuppressWarnings("unchecked")
|
239
|
+
public void testListFilesWithPathMatcher() throws Exception
|
240
|
+
{
|
241
|
+
final String pattern = FTP_TEST_PATH_PREFIX + "02";
|
242
|
+
final Pattern pathMatchPattern = Pattern.compile(pattern);
|
243
|
+
final List<String> expected = Arrays.asList(
|
244
|
+
FTP_TEST_PATH_PREFIX + "02.csv"
|
245
|
+
);
|
246
|
+
|
247
|
+
final ConfigSource config = config();
|
248
|
+
config.set("path_match_pattern", pattern);
|
152
249
|
final PluginTask task = config.loadConfig(PluginTask.class);
|
153
|
-
ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
250
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
154
251
|
@Override
|
155
|
-
public List<TaskReport> run(TaskSource taskSource, int taskCount)
|
252
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
156
253
|
{
|
157
|
-
|
254
|
+
assertEquals(taskCount, 1);
|
158
255
|
return emptyTaskReports(taskCount);
|
159
256
|
}
|
160
257
|
});
|
161
258
|
|
162
|
-
Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
|
259
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
163
260
|
method.setAccessible(true);
|
164
|
-
Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
165
|
-
List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
|
166
|
-
|
167
|
-
|
168
|
-
|
261
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
262
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
|
263
|
+
|
264
|
+
assertEquals(fileList.get(0), expected.get(0));
|
265
|
+
assertEquals(configDiff.get(String.class, "last_path"), FTP_TEST_PATH_PREFIX + "02.csv");
|
266
|
+
}
|
267
|
+
|
268
|
+
@Test
|
269
|
+
@SuppressWarnings("unchecked")
|
270
|
+
public void testListFilesWithNonExistPathMatcher() throws Exception
|
271
|
+
{
|
272
|
+
final String pattern = "non_exist_file_matcher";
|
273
|
+
final Pattern pathMatchPattern = Pattern.compile(pattern);
|
274
|
+
|
275
|
+
final ConfigSource config = config();
|
276
|
+
config.set("path_match_pattern", pattern);
|
277
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
278
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
279
|
+
@Override
|
280
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
281
|
+
{
|
282
|
+
assertEquals(taskCount, 0);
|
283
|
+
return emptyTaskReports(taskCount);
|
284
|
+
}
|
285
|
+
});
|
286
|
+
|
287
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
288
|
+
method.setAccessible(true);
|
289
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
290
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
|
291
|
+
|
292
|
+
assertEquals(fileList.size(), 0);
|
293
|
+
assertEquals(configDiff.get(String.class, "last_path"), "");
|
294
|
+
}
|
295
|
+
|
296
|
+
@Test
|
297
|
+
@SuppressWarnings("unchecked")
|
298
|
+
public void testListFilesWithNotExactPathMatcher() throws Exception
|
299
|
+
{
|
300
|
+
final String pattern = "\\b" + FTP_TEST_PATH_PREFIX + "\\b";
|
301
|
+
final Pattern pathMatchPattern = Pattern.compile(pattern);
|
302
|
+
|
303
|
+
final ConfigSource config = config();
|
304
|
+
config.set("path_match_pattern", pattern);
|
305
|
+
final PluginTask task = config.loadConfig(PluginTask.class);
|
306
|
+
final ConfigDiff configDiff = plugin.transaction(config, new FileInputPlugin.Control() {
|
307
|
+
@Override
|
308
|
+
public List<TaskReport> run(final TaskSource taskSource, final int taskCount)
|
309
|
+
{
|
310
|
+
assertEquals(taskCount, 0);
|
311
|
+
return emptyTaskReports(taskCount);
|
312
|
+
}
|
313
|
+
});
|
314
|
+
|
315
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
316
|
+
method.setAccessible(true);
|
317
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
318
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, pathMatchPattern);
|
319
|
+
|
320
|
+
assertEquals(fileList.size(), 0);
|
321
|
+
assertEquals(configDiff.get(String.class, "last_path"), "");
|
169
322
|
}
|
170
323
|
|
171
324
|
@Test
|
172
325
|
public void testListFilesByPrefixIncrementalFalse()
|
173
326
|
{
|
174
|
-
ConfigSource config = config()
|
327
|
+
final ConfigSource config = config()
|
175
328
|
.deepCopy()
|
176
329
|
.set("incremental", false);
|
177
330
|
|
178
|
-
ConfigDiff configDiff = runner.transaction(config, new Control());
|
331
|
+
final ConfigDiff configDiff = runner.transaction(config, new Control());
|
179
332
|
|
180
|
-
|
333
|
+
Assert.assertEquals(configDiff.toString(), "{}");
|
181
334
|
}
|
182
335
|
|
183
336
|
@Test
|
184
337
|
@SuppressWarnings("unchecked")
|
185
338
|
public void testFtpFileInputByOpen() throws Exception
|
186
339
|
{
|
187
|
-
ConfigSource config = config();
|
188
|
-
PluginTask task = config().loadConfig(PluginTask.class);
|
340
|
+
final ConfigSource config = config();
|
341
|
+
final PluginTask task = config().loadConfig(PluginTask.class);
|
189
342
|
|
190
343
|
runner.transaction(config, new Control());
|
191
344
|
|
192
|
-
Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class);
|
345
|
+
final Method method = FtpFileInputPlugin.class.getDeclaredMethod("listFiles", Logger.class, PluginTask.class, Pattern.class);
|
193
346
|
method.setAccessible(true);
|
194
|
-
Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
195
|
-
List<String> fileList = (List<String>) method.invoke(plugin, logger, task);
|
347
|
+
final Logger logger = Exec.getLogger(FtpFileInputPlugin.class);
|
348
|
+
final List<String> fileList = (List<String>) method.invoke(plugin, logger, task, defaultPathMatchPattern);
|
196
349
|
task.setFiles(fileList);
|
197
350
|
|
198
351
|
assertRecords(config, output);
|
199
352
|
}
|
200
353
|
|
201
|
-
private static List<TaskReport> emptyTaskReports(int taskCount)
|
354
|
+
private static List<TaskReport> emptyTaskReports(final int taskCount)
|
202
355
|
{
|
203
|
-
ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
356
|
+
final ImmutableList.Builder<TaskReport> reports = new ImmutableList.Builder<>();
|
204
357
|
for (int i = 0; i < taskCount; i++) {
|
205
358
|
reports.add(Exec.newTaskReport());
|
206
359
|
}
|
@@ -211,9 +364,9 @@ public class TestFtpFileInputPlugin
|
|
211
364
|
implements InputPlugin.Control
|
212
365
|
{
|
213
366
|
@Override
|
214
|
-
public List<TaskReport> run(TaskSource taskSource, Schema schema, int taskCount)
|
367
|
+
public List<TaskReport> run(final TaskSource taskSource, final Schema schema, final int taskCount)
|
215
368
|
{
|
216
|
-
List<TaskReport> reports = new ArrayList<>();
|
369
|
+
final List<TaskReport> reports = new ArrayList<>();
|
217
370
|
for (int i = 0; i < taskCount; i++) {
|
218
371
|
reports.add(runner.run(taskSource, schema, i, output));
|
219
372
|
}
|
@@ -239,9 +392,9 @@ public class TestFtpFileInputPlugin
|
|
239
392
|
.set("parser", parserConfig(schemaConfig()));
|
240
393
|
}
|
241
394
|
|
242
|
-
private ImmutableMap<String, Object> parserConfig(ImmutableList<Object> schemaConfig)
|
395
|
+
private ImmutableMap<String, Object> parserConfig(final ImmutableList<Object> schemaConfig)
|
243
396
|
{
|
244
|
-
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
397
|
+
final ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
|
245
398
|
builder.put("type", "csv");
|
246
399
|
builder.put("newline", "CRLF");
|
247
400
|
builder.put("delimiter", ",");
|
@@ -257,7 +410,7 @@ public class TestFtpFileInputPlugin
|
|
257
410
|
|
258
411
|
private ImmutableList<Object> schemaConfig()
|
259
412
|
{
|
260
|
-
ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
413
|
+
final ImmutableList.Builder<Object> builder = new ImmutableList.Builder<>();
|
261
414
|
builder.add(ImmutableMap.of("name", "id", "type", "long"));
|
262
415
|
builder.add(ImmutableMap.of("name", "account", "type", "long"));
|
263
416
|
builder.add(ImmutableMap.of("name", "time", "type", "timestamp", "format", "%Y-%m-%d %H:%M:%S"));
|
@@ -266,37 +419,37 @@ public class TestFtpFileInputPlugin
|
|
266
419
|
return builder.build();
|
267
420
|
}
|
268
421
|
|
269
|
-
public SSLPlugins.SSLPluginConfig sslConfig(PluginTask task)
|
422
|
+
public SSLPlugins.SSLPluginConfig sslConfig(final PluginTask task)
|
270
423
|
{
|
271
424
|
return SSLPlugins.configure(task);
|
272
425
|
}
|
273
426
|
|
274
|
-
private void assertRecords(ConfigSource config, MockPageOutput output)
|
427
|
+
private void assertRecords(final ConfigSource config, final MockPageOutput output)
|
275
428
|
{
|
276
|
-
List<Object[]> records = getRecords(config, output);
|
277
|
-
|
429
|
+
final List<Object[]> records = getRecords(config, output);
|
430
|
+
assertEquals(records.size(), 8);
|
278
431
|
{
|
279
|
-
Object[] record = records.get(0);
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
432
|
+
final Object[] record = records.get(0);
|
433
|
+
assertEquals((long) record[0], 1L);
|
434
|
+
assertEquals((long) record[1], 32864L);
|
435
|
+
assertEquals(record[2].toString(), "2015-01-27 19:23:49 UTC");
|
436
|
+
assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
|
437
|
+
assertEquals(record[4].toString(), "embulk");
|
285
438
|
}
|
286
439
|
|
287
440
|
{
|
288
|
-
Object[] record = records.get(1);
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
441
|
+
final Object[] record = records.get(1);
|
442
|
+
assertEquals((long) record[0], 2L);
|
443
|
+
assertEquals((long) record[1], 14824L);
|
444
|
+
assertEquals(record[2].toString(), "2015-01-27 19:01:23 UTC");
|
445
|
+
assertEquals(record[3].toString(), "2015-01-27 00:00:00 UTC");
|
446
|
+
assertEquals(record[4].toString(), "embulk jruby");
|
294
447
|
}
|
295
448
|
}
|
296
449
|
|
297
|
-
private List<Object[]> getRecords(ConfigSource config, MockPageOutput output)
|
450
|
+
private List<Object[]> getRecords(final ConfigSource config, final MockPageOutput output)
|
298
451
|
{
|
299
|
-
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
452
|
+
final Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
300
453
|
return Pages.toObjects(schema, output.pages);
|
301
454
|
}
|
302
455
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-input-ftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sadayuki Furuhashi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-11-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -52,10 +52,10 @@ files:
|
|
52
52
|
- src/test/resources/sample_01.csv
|
53
53
|
- src/test/resources/sample_02.csv
|
54
54
|
- classpath/bcpkix-jdk15on-1.52.jar
|
55
|
-
- classpath/embulk-util-ftp-0.2.
|
55
|
+
- classpath/embulk-util-ftp-0.2.1.jar
|
56
56
|
- classpath/bcprov-jdk15on-1.52.jar
|
57
57
|
- classpath/ftp4j-1.7.2.jar
|
58
|
-
- classpath/embulk-input-ftp-0.2.
|
58
|
+
- classpath/embulk-input-ftp-0.2.1.jar
|
59
59
|
homepage: https://github.com/embulk/embulk-input-ftp
|
60
60
|
licenses:
|
61
61
|
- Apache 2.0
|
Binary file
|