embulk-output-sftp 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -3
- data/CHANGELOG.md +3 -0
- data/README.md +3 -0
- data/build.gradle +6 -5
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +20 -3
- data/src/main/java/org/embulk/output/sftp/SftpLocalFileOutput.java +223 -0
- data/src/main/java/org/embulk/output/sftp/SftpRemoteFileOutput.java +120 -0
- data/src/main/java/org/embulk/output/sftp/SftpUtils.java +146 -123
- data/src/main/java/org/embulk/output/sftp/utils/DefaultRetry.java +55 -0
- data/src/main/java/org/embulk/output/sftp/utils/TimedCallable.java +27 -0
- data/src/main/java/org/embulk/output/sftp/utils/TimeoutCloser.java +42 -0
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +377 -8
- data/src/test/java/org/embulk/output/sftp/utils/TestTimedCallable.java +36 -0
- data/src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java +45 -0
- metadata +11 -5
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +0 -140
@@ -4,6 +4,9 @@ import com.google.common.base.Charsets;
|
|
4
4
|
import com.google.common.base.Optional;
|
5
5
|
import com.google.common.collect.Lists;
|
6
6
|
import com.google.common.io.Resources;
|
7
|
+
import com.jcraft.jsch.JSchException;
|
8
|
+
import org.apache.commons.vfs2.FileObject;
|
9
|
+
import org.apache.commons.vfs2.FileSystemException;
|
7
10
|
import org.apache.sshd.common.NamedFactory;
|
8
11
|
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
|
9
12
|
import org.apache.sshd.server.Command;
|
@@ -29,7 +32,9 @@ import org.embulk.spi.PageTestUtils;
|
|
29
32
|
import org.embulk.spi.Schema;
|
30
33
|
import org.embulk.spi.TransactionalPageOutput;
|
31
34
|
import org.embulk.spi.time.Timestamp;
|
35
|
+
import org.hamcrest.CoreMatchers;
|
32
36
|
import org.junit.After;
|
37
|
+
import org.junit.Assert;
|
33
38
|
import org.junit.Before;
|
34
39
|
import org.junit.Rule;
|
35
40
|
import org.junit.Test;
|
@@ -37,17 +42,28 @@ import org.junit.rules.ExpectedException;
|
|
37
42
|
import org.junit.rules.TemporaryFolder;
|
38
43
|
import org.littleshoot.proxy.HttpProxyServer;
|
39
44
|
import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
|
45
|
+
import org.mockito.Mockito;
|
46
|
+
import org.mockito.invocation.InvocationOnMock;
|
47
|
+
import org.mockito.stubbing.Answer;
|
40
48
|
import org.slf4j.Logger;
|
41
49
|
|
50
|
+
import java.io.BufferedOutputStream;
|
42
51
|
import java.io.File;
|
52
|
+
import java.io.FileInputStream;
|
53
|
+
import java.io.FileOutputStream;
|
43
54
|
import java.io.IOException;
|
55
|
+
import java.io.InputStream;
|
56
|
+
import java.io.OutputStream;
|
44
57
|
import java.nio.file.DirectoryStream;
|
45
58
|
import java.nio.file.Files;
|
46
59
|
import java.nio.file.Path;
|
47
60
|
import java.nio.file.Paths;
|
48
61
|
import java.security.PublicKey;
|
62
|
+
import java.util.Arrays;
|
49
63
|
import java.util.Collections;
|
50
64
|
import java.util.List;
|
65
|
+
import java.util.Random;
|
66
|
+
import java.util.concurrent.TimeoutException;
|
51
67
|
|
52
68
|
import static com.google.common.io.Files.readLines;
|
53
69
|
import static org.embulk.spi.type.Types.BOOLEAN;
|
@@ -58,8 +74,14 @@ import static org.embulk.spi.type.Types.STRING;
|
|
58
74
|
import static org.embulk.spi.type.Types.TIMESTAMP;
|
59
75
|
import static org.hamcrest.CoreMatchers.containsString;
|
60
76
|
import static org.hamcrest.CoreMatchers.hasItem;
|
77
|
+
import static org.junit.Assert.assertArrayEquals;
|
61
78
|
import static org.junit.Assert.assertEquals;
|
79
|
+
import static org.junit.Assert.assertFalse;
|
80
|
+
import static org.junit.Assert.assertNotNull;
|
81
|
+
import static org.junit.Assert.assertNull;
|
62
82
|
import static org.junit.Assert.assertThat;
|
83
|
+
import static org.junit.Assert.assertTrue;
|
84
|
+
import static org.junit.Assert.fail;
|
63
85
|
import static org.msgpack.value.ValueFactory.newMap;
|
64
86
|
import static org.msgpack.value.ValueFactory.newString;
|
65
87
|
|
@@ -94,6 +116,8 @@ public class TestSftpFileOutputPlugin
|
|
94
116
|
.add("_c5", JSON)
|
95
117
|
.build();
|
96
118
|
|
119
|
+
private final String defaultPathPrefix = "/test/testUserPassword";
|
120
|
+
|
97
121
|
@Before
|
98
122
|
public void createResources()
|
99
123
|
throws IOException
|
@@ -201,8 +225,8 @@ public class TestSftpFileOutputPlugin
|
|
201
225
|
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
202
226
|
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
|
203
227
|
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
|
204
|
-
|
205
|
-
|
228
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
|
229
|
+
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
|
206
230
|
pageOutput.add(page);
|
207
231
|
}
|
208
232
|
pageOutput.commit();
|
@@ -223,7 +247,7 @@ public class TestSftpFileOutputPlugin
|
|
223
247
|
{
|
224
248
|
try {
|
225
249
|
List<String> lines = readLines(new File(filePath),
|
226
|
-
|
250
|
+
Charsets.UTF_8);
|
227
251
|
for (int i = 0; i < lines.size(); i++) {
|
228
252
|
String[] record = lines.get(i).split(",");
|
229
253
|
if (i == 0) {
|
@@ -402,8 +426,8 @@ public class TestSftpFileOutputPlugin
|
|
402
426
|
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
403
427
|
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
404
428
|
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
405
|
-
|
406
|
-
|
429
|
+
testFolder.getRoot().getAbsolutePath(),
|
430
|
+
pathPrefix));
|
407
431
|
}
|
408
432
|
|
409
433
|
@Test
|
@@ -439,8 +463,8 @@ public class TestSftpFileOutputPlugin
|
|
439
463
|
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
440
464
|
|
441
465
|
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
442
|
-
|
443
|
-
|
466
|
+
testFolder.getRoot().getAbsolutePath(),
|
467
|
+
pathPrefix));
|
444
468
|
}
|
445
469
|
|
446
470
|
@Test
|
@@ -448,7 +472,7 @@ public class TestSftpFileOutputPlugin
|
|
448
472
|
{
|
449
473
|
HttpProxyServer proxyServer = null;
|
450
474
|
try {
|
451
|
-
|
475
|
+
proxyServer = createProxyServer(PROXY_PORT);
|
452
476
|
|
453
477
|
// setting embulk config
|
454
478
|
final String pathPrefix = "/test/testUserPassword";
|
@@ -522,4 +546,349 @@ public class TestSftpFileOutputPlugin
|
|
522
546
|
assertEquals(ConfigException.class, e.getClass());
|
523
547
|
}
|
524
548
|
}
|
549
|
+
|
550
|
+
@Test
|
551
|
+
public void testUploadFileWithRetry() throws IOException
|
552
|
+
{
|
553
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
554
|
+
SftpUtils utils = Mockito.spy(new SftpUtils(task));
|
555
|
+
|
556
|
+
// append throws exception
|
557
|
+
Mockito.doThrow(new IOException("Fake Exception"))
|
558
|
+
.doCallRealMethod()
|
559
|
+
.when(utils)
|
560
|
+
.appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
561
|
+
|
562
|
+
byte[] expected = randBytes(8);
|
563
|
+
File input = writeBytesToInputFile(expected);
|
564
|
+
utils.uploadFile(input, defaultPathPrefix);
|
565
|
+
|
566
|
+
// assert retry and recover
|
567
|
+
Mockito.verify(utils, Mockito.times(2)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
568
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
569
|
+
assertThat(fileList, hasItem(containsString(defaultPathPrefix)));
|
570
|
+
|
571
|
+
// assert uploaded file
|
572
|
+
String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + defaultPathPrefix;
|
573
|
+
File output = new File(filePath);
|
574
|
+
InputStream in = new FileInputStream(output);
|
575
|
+
byte[] actual = new byte[8];
|
576
|
+
in.read(actual);
|
577
|
+
in.close();
|
578
|
+
Assert.assertArrayEquals(expected, actual);
|
579
|
+
}
|
580
|
+
|
581
|
+
@Test
|
582
|
+
public void testUploadFileRetryAndGiveUp() throws IOException
|
583
|
+
{
|
584
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
585
|
+
SftpUtils utils = Mockito.spy(new SftpUtils(task));
|
586
|
+
|
587
|
+
// append throws exception
|
588
|
+
Mockito.doThrow(new IOException("Fake IOException"))
|
589
|
+
.when(utils)
|
590
|
+
.appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
591
|
+
|
592
|
+
byte[] expected = randBytes(8);
|
593
|
+
File input = writeBytesToInputFile(expected);
|
594
|
+
try {
|
595
|
+
utils.uploadFile(input, defaultPathPrefix);
|
596
|
+
fail("Should not reach here");
|
597
|
+
}
|
598
|
+
catch (Exception e) {
|
599
|
+
assertThat(e, CoreMatchers.<Exception>instanceOf(RuntimeException.class));
|
600
|
+
assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(IOException.class));
|
601
|
+
assertEquals(e.getCause().getMessage(), "Fake IOException");
|
602
|
+
// assert used up all retries
|
603
|
+
Mockito.verify(utils, Mockito.times(task.getMaxConnectionRetry() + 1)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
604
|
+
assertEmptyUploadedFile(defaultPathPrefix);
|
605
|
+
}
|
606
|
+
}
|
607
|
+
|
608
|
+
@Test
|
609
|
+
public void testUploadFileNotRetryAuthFail() throws IOException
|
610
|
+
{
|
611
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
612
|
+
SftpUtils utils = Mockito.spy(new SftpUtils(task));
|
613
|
+
|
614
|
+
// append throws exception
|
615
|
+
Mockito.doThrow(new IOException(new JSchException("USERAUTH fail")))
|
616
|
+
.doCallRealMethod()
|
617
|
+
.when(utils)
|
618
|
+
.appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
619
|
+
|
620
|
+
byte[] expected = randBytes(8);
|
621
|
+
File input = writeBytesToInputFile(expected);
|
622
|
+
try {
|
623
|
+
utils.uploadFile(input, defaultPathPrefix);
|
624
|
+
fail("Should not reach here");
|
625
|
+
}
|
626
|
+
catch (Exception e) {
|
627
|
+
assertThat(e, CoreMatchers.<Exception>instanceOf(RuntimeException.class));
|
628
|
+
assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(IOException.class));
|
629
|
+
assertThat(e.getCause().getCause(), CoreMatchers.<Throwable>instanceOf(JSchException.class));
|
630
|
+
assertEquals(e.getCause().getCause().getMessage(), "USERAUTH fail");
|
631
|
+
// assert no retry
|
632
|
+
Mockito.verify(utils, Mockito.times(1)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
|
633
|
+
assertEmptyUploadedFile(defaultPathPrefix);
|
634
|
+
}
|
635
|
+
}
|
636
|
+
|
637
|
+
@Test
|
638
|
+
public void testAppendFile() throws IOException
|
639
|
+
{
|
640
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
641
|
+
SftpUtils utils = new SftpUtils(task);
|
642
|
+
|
643
|
+
FileObject remoteFile = utils.resolve(defaultPathPrefix);
|
644
|
+
BufferedOutputStream remoteOutput = utils.openStream(remoteFile);
|
645
|
+
// 1st append
|
646
|
+
byte[] expected = randBytes(16);
|
647
|
+
utils.appendFile(writeBytesToInputFile(Arrays.copyOfRange(expected, 0, 8)), remoteFile, remoteOutput);
|
648
|
+
// 2nd append
|
649
|
+
utils.appendFile(writeBytesToInputFile(Arrays.copyOfRange(expected, 8, 16)), remoteFile, remoteOutput);
|
650
|
+
remoteOutput.close();
|
651
|
+
remoteFile.close();
|
652
|
+
|
653
|
+
// assert uploaded file
|
654
|
+
String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + defaultPathPrefix;
|
655
|
+
File output = new File(filePath);
|
656
|
+
InputStream in = new FileInputStream(output);
|
657
|
+
byte[] actual = new byte[16];
|
658
|
+
in.read(actual);
|
659
|
+
in.close();
|
660
|
+
assertArrayEquals(expected, actual);
|
661
|
+
}
|
662
|
+
|
663
|
+
@Test
|
664
|
+
public void testAppendFileAndTimeOut() throws IOException
|
665
|
+
{
|
666
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
667
|
+
SftpUtils utils = new SftpUtils(task);
|
668
|
+
utils.writeTimeout = 1; // 1s time-out
|
669
|
+
|
670
|
+
byte[] expected = randBytes(8);
|
671
|
+
FileObject remoteFile = utils.resolve(defaultPathPrefix);
|
672
|
+
BufferedOutputStream remoteOutput = Mockito.spy(utils.openStream(remoteFile));
|
673
|
+
|
674
|
+
Mockito.doAnswer(new Answer()
|
675
|
+
{
|
676
|
+
@Override
|
677
|
+
public Void answer(InvocationOnMock invocation) throws Throwable
|
678
|
+
{
|
679
|
+
Thread.sleep(2000); // 2s
|
680
|
+
return null;
|
681
|
+
}
|
682
|
+
}).when(remoteOutput).write(Mockito.any(byte[].class), Mockito.eq(0), Mockito.eq(8));
|
683
|
+
|
684
|
+
try {
|
685
|
+
utils.appendFile(writeBytesToInputFile(expected), remoteFile, remoteOutput);
|
686
|
+
fail("Should not reach here");
|
687
|
+
}
|
688
|
+
catch (IOException e) {
|
689
|
+
assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(TimeoutException.class));
|
690
|
+
assertNull(e.getCause().getMessage());
|
691
|
+
}
|
692
|
+
}
|
693
|
+
|
694
|
+
@Test
|
695
|
+
public void testRenameFileWithRetry() throws IOException
|
696
|
+
{
|
697
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
698
|
+
SftpUtils utils = Mockito.spy(new SftpUtils(task));
|
699
|
+
|
700
|
+
byte[] expected = randBytes(8);
|
701
|
+
File input = writeBytesToInputFile(expected);
|
702
|
+
utils.uploadFile(input, defaultPathPrefix);
|
703
|
+
|
704
|
+
Mockito.doThrow(new RuntimeException("Fake Exception"))
|
705
|
+
.doCallRealMethod()
|
706
|
+
.when(utils).resolve(Mockito.eq(defaultPathPrefix));
|
707
|
+
|
708
|
+
utils.renameFile(defaultPathPrefix, "/after");
|
709
|
+
Mockito.verify(utils, Mockito.times(1 + 2)).resolve(Mockito.any(String.class)); // 1 fail + 2 success
|
710
|
+
|
711
|
+
// assert renamed file
|
712
|
+
String filePath = testFolder.getRoot().getAbsolutePath() + "/after";
|
713
|
+
File output = new File(filePath);
|
714
|
+
InputStream in = new FileInputStream(output);
|
715
|
+
byte[] actual = new byte[8];
|
716
|
+
in.read(actual);
|
717
|
+
in.close();
|
718
|
+
Assert.assertArrayEquals(expected, actual);
|
719
|
+
}
|
720
|
+
|
721
|
+
@Test
|
722
|
+
public void testDeleteFile() throws IOException
|
723
|
+
{
|
724
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
725
|
+
SftpUtils utils = new SftpUtils(task);
|
726
|
+
|
727
|
+
// upload file
|
728
|
+
byte[] expected = randBytes(8);
|
729
|
+
File input = writeBytesToInputFile(expected);
|
730
|
+
utils.uploadFile(input, defaultPathPrefix);
|
731
|
+
|
732
|
+
FileObject target = utils.resolve(defaultPathPrefix);
|
733
|
+
assertTrue("File should exists", target.exists());
|
734
|
+
|
735
|
+
utils.deleteFile(defaultPathPrefix);
|
736
|
+
|
737
|
+
target = utils.resolve(defaultPathPrefix);
|
738
|
+
assertFalse("File should be deleted", target.exists());
|
739
|
+
}
|
740
|
+
|
741
|
+
@Test
|
742
|
+
public void testDeleteFileNotExists()
|
743
|
+
{
|
744
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
745
|
+
SftpUtils utils = new SftpUtils(task);
|
746
|
+
utils.deleteFile("/not/exists");
|
747
|
+
}
|
748
|
+
|
749
|
+
@Test
|
750
|
+
public void testResolveWithoutRetry()
|
751
|
+
{
|
752
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
753
|
+
SftpUtils utils = Mockito.spy(new SftpUtils(task));
|
754
|
+
|
755
|
+
Mockito.doThrow(new ConfigException("Fake ConfigException"))
|
756
|
+
.doCallRealMethod()
|
757
|
+
.when(utils).getSftpFileUri(Mockito.eq(defaultPathPrefix));
|
758
|
+
|
759
|
+
try {
|
760
|
+
utils.resolve(defaultPathPrefix);
|
761
|
+
fail("Should not reach here");
|
762
|
+
}
|
763
|
+
catch (Exception e) {
|
764
|
+
assertThat(e, CoreMatchers.<Exception>instanceOf(ConfigException.class));
|
765
|
+
// assert retry
|
766
|
+
Mockito.verify(utils, Mockito.times(1)).getSftpFileUri(Mockito.eq(defaultPathPrefix));
|
767
|
+
}
|
768
|
+
}
|
769
|
+
|
770
|
+
@Test
|
771
|
+
public void testOpenStreamWithRetry() throws FileSystemException
|
772
|
+
{
|
773
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
774
|
+
SftpUtils utils = new SftpUtils(task);
|
775
|
+
|
776
|
+
FileObject mock = Mockito.spy(utils.resolve(defaultPathPrefix));
|
777
|
+
Mockito.doThrow(new FileSystemException("Fake FileSystemException"))
|
778
|
+
.doCallRealMethod()
|
779
|
+
.when(mock).getContent();
|
780
|
+
|
781
|
+
OutputStream stream = utils.openStream(mock);
|
782
|
+
assertNotNull(stream);
|
783
|
+
Mockito.verify(mock, Mockito.times(2)).getContent();
|
784
|
+
}
|
785
|
+
|
786
|
+
@Test
|
787
|
+
public void testNewSftpFileExists() throws IOException
|
788
|
+
{
|
789
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
790
|
+
SftpUtils utils = new SftpUtils(task);
|
791
|
+
|
792
|
+
byte[] expected = randBytes(8);
|
793
|
+
File input = writeBytesToInputFile(expected);
|
794
|
+
utils.uploadFile(input, defaultPathPrefix);
|
795
|
+
|
796
|
+
FileObject file = utils.resolve(defaultPathPrefix);
|
797
|
+
assertTrue("File should exists", file.exists());
|
798
|
+
|
799
|
+
file = utils.newSftpFile(utils.getSftpFileUri(defaultPathPrefix));
|
800
|
+
assertFalse("File should be deleted", file.exists());
|
801
|
+
}
|
802
|
+
|
803
|
+
@Test
|
804
|
+
public void testNewSftpFileParentNotExists() throws FileSystemException
|
805
|
+
{
|
806
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
807
|
+
SftpUtils utils = new SftpUtils(task);
|
808
|
+
|
809
|
+
String parentPath = defaultPathPrefix.substring(0, defaultPathPrefix.lastIndexOf('/'));
|
810
|
+
FileObject parent = utils.resolve(parentPath);
|
811
|
+
boolean exists = parent.exists();
|
812
|
+
logger.info("Resolving parent path: {}, exists? {}", parentPath, exists);
|
813
|
+
assertFalse("Parent folder should not exist", exists);
|
814
|
+
|
815
|
+
utils.newSftpFile(utils.getSftpFileUri(defaultPathPrefix));
|
816
|
+
|
817
|
+
parent = utils.resolve(parentPath);
|
818
|
+
exists = parent.exists();
|
819
|
+
logger.info("Resolving (again) parent path: {}, exists? {}", parentPath, exists);
|
820
|
+
assertTrue("Parent folder must be created", parent.exists());
|
821
|
+
}
|
822
|
+
|
823
|
+
@Test
|
824
|
+
public void testSftpFileOutputNextFile()
|
825
|
+
{
|
826
|
+
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
827
|
+
|
828
|
+
SftpLocalFileOutput localFileOutput = new SftpLocalFileOutput(task, 1);
|
829
|
+
localFileOutput.nextFile();
|
830
|
+
assertNotNull("Must use local temp file", localFileOutput.getLocalOutput());
|
831
|
+
assertNull("Must not use remote temp file", localFileOutput.getRemoteOutput());
|
832
|
+
localFileOutput.close();
|
833
|
+
|
834
|
+
SftpRemoteFileOutput remoteFileOutput = new SftpRemoteFileOutput(task, 1);
|
835
|
+
remoteFileOutput.nextFile();
|
836
|
+
assertNull("Must not use local temp file", remoteFileOutput.getLocalOutput());
|
837
|
+
assertNotNull("Must use remote temp file", remoteFileOutput.getRemoteOutput());
|
838
|
+
remoteFileOutput.close();
|
839
|
+
}
|
840
|
+
|
841
|
+
private String defaultConfig(final String pathPrefix)
|
842
|
+
{
|
843
|
+
return "type: sftp\n" +
|
844
|
+
"host: " + HOST + "\n" +
|
845
|
+
"port: " + PORT + "\n" +
|
846
|
+
"user: " + USERNAME + "\n" +
|
847
|
+
"password: " + PASSWORD + "\n" +
|
848
|
+
"path_prefix: " + pathPrefix + "\n" +
|
849
|
+
"file_ext: txt\n" +
|
850
|
+
"formatter:\n" +
|
851
|
+
" type: csv\n" +
|
852
|
+
" newline: CRLF\n" +
|
853
|
+
" newline_in_field: LF\n" +
|
854
|
+
" header_line: true\n" +
|
855
|
+
" charset: UTF-8\n" +
|
856
|
+
" quote_policy: NONE\n" +
|
857
|
+
" quote: \"\\\"\"\n" +
|
858
|
+
" escape: \"\\\\\"\n" +
|
859
|
+
" null_string: \"\"\n" +
|
860
|
+
" default_timezone: 'UTC'";
|
861
|
+
}
|
862
|
+
|
863
|
+
private PluginTask defaultTask()
|
864
|
+
{
|
865
|
+
ConfigSource config = getConfigFromYaml(defaultConfig(defaultPathPrefix));
|
866
|
+
return config.loadConfig(SftpFileOutputPlugin.PluginTask.class);
|
867
|
+
}
|
868
|
+
|
869
|
+
private byte[] randBytes(final int len)
|
870
|
+
{
|
871
|
+
byte[] bytes = new byte[len];
|
872
|
+
new Random().nextBytes(bytes);
|
873
|
+
return bytes;
|
874
|
+
}
|
875
|
+
|
876
|
+
private File writeBytesToInputFile(final byte[] expected) throws IOException
|
877
|
+
{
|
878
|
+
File input = File.createTempFile("anything", ".dat");
|
879
|
+
OutputStream out = new BufferedOutputStream(new FileOutputStream(input));
|
880
|
+
out.write(expected);
|
881
|
+
out.close();
|
882
|
+
return input;
|
883
|
+
}
|
884
|
+
|
885
|
+
private void assertEmptyUploadedFile(final String pathPrefix)
|
886
|
+
{
|
887
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
888
|
+
assertThat(fileList, hasItem(containsString(pathPrefix)));
|
889
|
+
|
890
|
+
String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + pathPrefix;
|
891
|
+
File output = new File(filePath);
|
892
|
+
assertEquals(0, output.length());
|
893
|
+
}
|
525
894
|
}
|