embulk-output-sftp 0.1.10 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,6 +4,9 @@ import com.google.common.base.Charsets;
4
4
  import com.google.common.base.Optional;
5
5
  import com.google.common.collect.Lists;
6
6
  import com.google.common.io.Resources;
7
+ import com.jcraft.jsch.JSchException;
8
+ import org.apache.commons.vfs2.FileObject;
9
+ import org.apache.commons.vfs2.FileSystemException;
7
10
  import org.apache.sshd.common.NamedFactory;
8
11
  import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
9
12
  import org.apache.sshd.server.Command;
@@ -29,7 +32,9 @@ import org.embulk.spi.PageTestUtils;
29
32
  import org.embulk.spi.Schema;
30
33
  import org.embulk.spi.TransactionalPageOutput;
31
34
  import org.embulk.spi.time.Timestamp;
35
+ import org.hamcrest.CoreMatchers;
32
36
  import org.junit.After;
37
+ import org.junit.Assert;
33
38
  import org.junit.Before;
34
39
  import org.junit.Rule;
35
40
  import org.junit.Test;
@@ -37,17 +42,28 @@ import org.junit.rules.ExpectedException;
37
42
  import org.junit.rules.TemporaryFolder;
38
43
  import org.littleshoot.proxy.HttpProxyServer;
39
44
  import org.littleshoot.proxy.impl.DefaultHttpProxyServer;
45
+ import org.mockito.Mockito;
46
+ import org.mockito.invocation.InvocationOnMock;
47
+ import org.mockito.stubbing.Answer;
40
48
  import org.slf4j.Logger;
41
49
 
50
+ import java.io.BufferedOutputStream;
42
51
  import java.io.File;
52
+ import java.io.FileInputStream;
53
+ import java.io.FileOutputStream;
43
54
  import java.io.IOException;
55
+ import java.io.InputStream;
56
+ import java.io.OutputStream;
44
57
  import java.nio.file.DirectoryStream;
45
58
  import java.nio.file.Files;
46
59
  import java.nio.file.Path;
47
60
  import java.nio.file.Paths;
48
61
  import java.security.PublicKey;
62
+ import java.util.Arrays;
49
63
  import java.util.Collections;
50
64
  import java.util.List;
65
+ import java.util.Random;
66
+ import java.util.concurrent.TimeoutException;
51
67
 
52
68
  import static com.google.common.io.Files.readLines;
53
69
  import static org.embulk.spi.type.Types.BOOLEAN;
@@ -58,8 +74,14 @@ import static org.embulk.spi.type.Types.STRING;
58
74
  import static org.embulk.spi.type.Types.TIMESTAMP;
59
75
  import static org.hamcrest.CoreMatchers.containsString;
60
76
  import static org.hamcrest.CoreMatchers.hasItem;
77
+ import static org.junit.Assert.assertArrayEquals;
61
78
  import static org.junit.Assert.assertEquals;
79
+ import static org.junit.Assert.assertFalse;
80
+ import static org.junit.Assert.assertNotNull;
81
+ import static org.junit.Assert.assertNull;
62
82
  import static org.junit.Assert.assertThat;
83
+ import static org.junit.Assert.assertTrue;
84
+ import static org.junit.Assert.fail;
63
85
  import static org.msgpack.value.ValueFactory.newMap;
64
86
  import static org.msgpack.value.ValueFactory.newString;
65
87
 
@@ -94,6 +116,8 @@ public class TestSftpFileOutputPlugin
94
116
  .add("_c5", JSON)
95
117
  .build();
96
118
 
119
+ private final String defaultPathPrefix = "/test/testUserPassword";
120
+
97
121
  @Before
98
122
  public void createResources()
99
123
  throws IOException
@@ -201,8 +225,8 @@ public class TestSftpFileOutputPlugin
201
225
  // true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
202
226
  // true,2,3.0,45,1970-01-01 00:00:00.678000 +0000,{\"k\":\"v\"}
203
227
  for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA,
204
- true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
205
- true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
228
+ true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
229
+ true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
206
230
  pageOutput.add(page);
207
231
  }
208
232
  pageOutput.commit();
@@ -223,7 +247,7 @@ public class TestSftpFileOutputPlugin
223
247
  {
224
248
  try {
225
249
  List<String> lines = readLines(new File(filePath),
226
- Charsets.UTF_8);
250
+ Charsets.UTF_8);
227
251
  for (int i = 0; i < lines.size(); i++) {
228
252
  String[] record = lines.get(i).split(",");
229
253
  if (i == 0) {
@@ -402,8 +426,8 @@ public class TestSftpFileOutputPlugin
402
426
  List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
403
427
  assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
404
428
  assertRecordsInFile(String.format("%s/%s001.00.txt",
405
- testFolder.getRoot().getAbsolutePath(),
406
- pathPrefix));
429
+ testFolder.getRoot().getAbsolutePath(),
430
+ pathPrefix));
407
431
  }
408
432
 
409
433
  @Test
@@ -439,8 +463,8 @@ public class TestSftpFileOutputPlugin
439
463
  assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
440
464
 
441
465
  assertRecordsInFile(String.format("%s/%s001.00.txt",
442
- testFolder.getRoot().getAbsolutePath(),
443
- pathPrefix));
466
+ testFolder.getRoot().getAbsolutePath(),
467
+ pathPrefix));
444
468
  }
445
469
 
446
470
  @Test
@@ -448,7 +472,7 @@ public class TestSftpFileOutputPlugin
448
472
  {
449
473
  HttpProxyServer proxyServer = null;
450
474
  try {
451
- proxyServer = createProxyServer(PROXY_PORT);
475
+ proxyServer = createProxyServer(PROXY_PORT);
452
476
 
453
477
  // setting embulk config
454
478
  final String pathPrefix = "/test/testUserPassword";
@@ -522,4 +546,349 @@ public class TestSftpFileOutputPlugin
522
546
  assertEquals(ConfigException.class, e.getClass());
523
547
  }
524
548
  }
549
+
550
+ @Test
551
+ public void testUploadFileWithRetry() throws IOException
552
+ {
553
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
554
+ SftpUtils utils = Mockito.spy(new SftpUtils(task));
555
+
556
+ // append throws exception
557
+ Mockito.doThrow(new IOException("Fake Exception"))
558
+ .doCallRealMethod()
559
+ .when(utils)
560
+ .appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
561
+
562
+ byte[] expected = randBytes(8);
563
+ File input = writeBytesToInputFile(expected);
564
+ utils.uploadFile(input, defaultPathPrefix);
565
+
566
+ // assert retry and recover
567
+ Mockito.verify(utils, Mockito.times(2)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
568
+ List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
569
+ assertThat(fileList, hasItem(containsString(defaultPathPrefix)));
570
+
571
+ // assert uploaded file
572
+ String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + defaultPathPrefix;
573
+ File output = new File(filePath);
574
+ InputStream in = new FileInputStream(output);
575
+ byte[] actual = new byte[8];
576
+ in.read(actual);
577
+ in.close();
578
+ Assert.assertArrayEquals(expected, actual);
579
+ }
580
+
581
+ @Test
582
+ public void testUploadFileRetryAndGiveUp() throws IOException
583
+ {
584
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
585
+ SftpUtils utils = Mockito.spy(new SftpUtils(task));
586
+
587
+ // append throws exception
588
+ Mockito.doThrow(new IOException("Fake IOException"))
589
+ .when(utils)
590
+ .appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
591
+
592
+ byte[] expected = randBytes(8);
593
+ File input = writeBytesToInputFile(expected);
594
+ try {
595
+ utils.uploadFile(input, defaultPathPrefix);
596
+ fail("Should not reach here");
597
+ }
598
+ catch (Exception e) {
599
+ assertThat(e, CoreMatchers.<Exception>instanceOf(RuntimeException.class));
600
+ assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(IOException.class));
601
+ assertEquals(e.getCause().getMessage(), "Fake IOException");
602
+ // assert used up all retries
603
+ Mockito.verify(utils, Mockito.times(task.getMaxConnectionRetry() + 1)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
604
+ assertEmptyUploadedFile(defaultPathPrefix);
605
+ }
606
+ }
607
+
608
+ @Test
609
+ public void testUploadFileNotRetryAuthFail() throws IOException
610
+ {
611
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
612
+ SftpUtils utils = Mockito.spy(new SftpUtils(task));
613
+
614
+ // append throws exception
615
+ Mockito.doThrow(new IOException(new JSchException("USERAUTH fail")))
616
+ .doCallRealMethod()
617
+ .when(utils)
618
+ .appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
619
+
620
+ byte[] expected = randBytes(8);
621
+ File input = writeBytesToInputFile(expected);
622
+ try {
623
+ utils.uploadFile(input, defaultPathPrefix);
624
+ fail("Should not reach here");
625
+ }
626
+ catch (Exception e) {
627
+ assertThat(e, CoreMatchers.<Exception>instanceOf(RuntimeException.class));
628
+ assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(IOException.class));
629
+ assertThat(e.getCause().getCause(), CoreMatchers.<Throwable>instanceOf(JSchException.class));
630
+ assertEquals(e.getCause().getCause().getMessage(), "USERAUTH fail");
631
+ // assert no retry
632
+ Mockito.verify(utils, Mockito.times(1)).appendFile(Mockito.any(File.class), Mockito.any(FileObject.class), Mockito.any(BufferedOutputStream.class));
633
+ assertEmptyUploadedFile(defaultPathPrefix);
634
+ }
635
+ }
636
+
637
+ @Test
638
+ public void testAppendFile() throws IOException
639
+ {
640
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
641
+ SftpUtils utils = new SftpUtils(task);
642
+
643
+ FileObject remoteFile = utils.resolve(defaultPathPrefix);
644
+ BufferedOutputStream remoteOutput = utils.openStream(remoteFile);
645
+ // 1st append
646
+ byte[] expected = randBytes(16);
647
+ utils.appendFile(writeBytesToInputFile(Arrays.copyOfRange(expected, 0, 8)), remoteFile, remoteOutput);
648
+ // 2nd append
649
+ utils.appendFile(writeBytesToInputFile(Arrays.copyOfRange(expected, 8, 16)), remoteFile, remoteOutput);
650
+ remoteOutput.close();
651
+ remoteFile.close();
652
+
653
+ // assert uploaded file
654
+ String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + defaultPathPrefix;
655
+ File output = new File(filePath);
656
+ InputStream in = new FileInputStream(output);
657
+ byte[] actual = new byte[16];
658
+ in.read(actual);
659
+ in.close();
660
+ assertArrayEquals(expected, actual);
661
+ }
662
+
663
+ @Test
664
+ public void testAppendFileAndTimeOut() throws IOException
665
+ {
666
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
667
+ SftpUtils utils = new SftpUtils(task);
668
+ utils.writeTimeout = 1; // 1s time-out
669
+
670
+ byte[] expected = randBytes(8);
671
+ FileObject remoteFile = utils.resolve(defaultPathPrefix);
672
+ BufferedOutputStream remoteOutput = Mockito.spy(utils.openStream(remoteFile));
673
+
674
+ Mockito.doAnswer(new Answer()
675
+ {
676
+ @Override
677
+ public Void answer(InvocationOnMock invocation) throws Throwable
678
+ {
679
+ Thread.sleep(2000); // 2s
680
+ return null;
681
+ }
682
+ }).when(remoteOutput).write(Mockito.any(byte[].class), Mockito.eq(0), Mockito.eq(8));
683
+
684
+ try {
685
+ utils.appendFile(writeBytesToInputFile(expected), remoteFile, remoteOutput);
686
+ fail("Should not reach here");
687
+ }
688
+ catch (IOException e) {
689
+ assertThat(e.getCause(), CoreMatchers.<Throwable>instanceOf(TimeoutException.class));
690
+ assertNull(e.getCause().getMessage());
691
+ }
692
+ }
693
+
694
+ @Test
695
+ public void testRenameFileWithRetry() throws IOException
696
+ {
697
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
698
+ SftpUtils utils = Mockito.spy(new SftpUtils(task));
699
+
700
+ byte[] expected = randBytes(8);
701
+ File input = writeBytesToInputFile(expected);
702
+ utils.uploadFile(input, defaultPathPrefix);
703
+
704
+ Mockito.doThrow(new RuntimeException("Fake Exception"))
705
+ .doCallRealMethod()
706
+ .when(utils).resolve(Mockito.eq(defaultPathPrefix));
707
+
708
+ utils.renameFile(defaultPathPrefix, "/after");
709
+ Mockito.verify(utils, Mockito.times(1 + 2)).resolve(Mockito.any(String.class)); // 1 fail + 2 success
710
+
711
+ // assert renamed file
712
+ String filePath = testFolder.getRoot().getAbsolutePath() + "/after";
713
+ File output = new File(filePath);
714
+ InputStream in = new FileInputStream(output);
715
+ byte[] actual = new byte[8];
716
+ in.read(actual);
717
+ in.close();
718
+ Assert.assertArrayEquals(expected, actual);
719
+ }
720
+
721
+ @Test
722
+ public void testDeleteFile() throws IOException
723
+ {
724
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
725
+ SftpUtils utils = new SftpUtils(task);
726
+
727
+ // upload file
728
+ byte[] expected = randBytes(8);
729
+ File input = writeBytesToInputFile(expected);
730
+ utils.uploadFile(input, defaultPathPrefix);
731
+
732
+ FileObject target = utils.resolve(defaultPathPrefix);
733
+ assertTrue("File should exists", target.exists());
734
+
735
+ utils.deleteFile(defaultPathPrefix);
736
+
737
+ target = utils.resolve(defaultPathPrefix);
738
+ assertFalse("File should be deleted", target.exists());
739
+ }
740
+
741
+ @Test
742
+ public void testDeleteFileNotExists()
743
+ {
744
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
745
+ SftpUtils utils = new SftpUtils(task);
746
+ utils.deleteFile("/not/exists");
747
+ }
748
+
749
+ @Test
750
+ public void testResolveWithoutRetry()
751
+ {
752
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
753
+ SftpUtils utils = Mockito.spy(new SftpUtils(task));
754
+
755
+ Mockito.doThrow(new ConfigException("Fake ConfigException"))
756
+ .doCallRealMethod()
757
+ .when(utils).getSftpFileUri(Mockito.eq(defaultPathPrefix));
758
+
759
+ try {
760
+ utils.resolve(defaultPathPrefix);
761
+ fail("Should not reach here");
762
+ }
763
+ catch (Exception e) {
764
+ assertThat(e, CoreMatchers.<Exception>instanceOf(ConfigException.class));
765
+ // assert retry
766
+ Mockito.verify(utils, Mockito.times(1)).getSftpFileUri(Mockito.eq(defaultPathPrefix));
767
+ }
768
+ }
769
+
770
+ @Test
771
+ public void testOpenStreamWithRetry() throws FileSystemException
772
+ {
773
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
774
+ SftpUtils utils = new SftpUtils(task);
775
+
776
+ FileObject mock = Mockito.spy(utils.resolve(defaultPathPrefix));
777
+ Mockito.doThrow(new FileSystemException("Fake FileSystemException"))
778
+ .doCallRealMethod()
779
+ .when(mock).getContent();
780
+
781
+ OutputStream stream = utils.openStream(mock);
782
+ assertNotNull(stream);
783
+ Mockito.verify(mock, Mockito.times(2)).getContent();
784
+ }
785
+
786
+ @Test
787
+ public void testNewSftpFileExists() throws IOException
788
+ {
789
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
790
+ SftpUtils utils = new SftpUtils(task);
791
+
792
+ byte[] expected = randBytes(8);
793
+ File input = writeBytesToInputFile(expected);
794
+ utils.uploadFile(input, defaultPathPrefix);
795
+
796
+ FileObject file = utils.resolve(defaultPathPrefix);
797
+ assertTrue("File should exists", file.exists());
798
+
799
+ file = utils.newSftpFile(utils.getSftpFileUri(defaultPathPrefix));
800
+ assertFalse("File should be deleted", file.exists());
801
+ }
802
+
803
+ @Test
804
+ public void testNewSftpFileParentNotExists() throws FileSystemException
805
+ {
806
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
807
+ SftpUtils utils = new SftpUtils(task);
808
+
809
+ String parentPath = defaultPathPrefix.substring(0, defaultPathPrefix.lastIndexOf('/'));
810
+ FileObject parent = utils.resolve(parentPath);
811
+ boolean exists = parent.exists();
812
+ logger.info("Resolving parent path: {}, exists? {}", parentPath, exists);
813
+ assertFalse("Parent folder should not exist", exists);
814
+
815
+ utils.newSftpFile(utils.getSftpFileUri(defaultPathPrefix));
816
+
817
+ parent = utils.resolve(parentPath);
818
+ exists = parent.exists();
819
+ logger.info("Resolving (again) parent path: {}, exists? {}", parentPath, exists);
820
+ assertTrue("Parent folder must be created", parent.exists());
821
+ }
822
+
823
+ @Test
824
+ public void testSftpFileOutputNextFile()
825
+ {
826
+ SftpFileOutputPlugin.PluginTask task = defaultTask();
827
+
828
+ SftpLocalFileOutput localFileOutput = new SftpLocalFileOutput(task, 1);
829
+ localFileOutput.nextFile();
830
+ assertNotNull("Must use local temp file", localFileOutput.getLocalOutput());
831
+ assertNull("Must not use remote temp file", localFileOutput.getRemoteOutput());
832
+ localFileOutput.close();
833
+
834
+ SftpRemoteFileOutput remoteFileOutput = new SftpRemoteFileOutput(task, 1);
835
+ remoteFileOutput.nextFile();
836
+ assertNull("Must not use local temp file", remoteFileOutput.getLocalOutput());
837
+ assertNotNull("Must use remote temp file", remoteFileOutput.getRemoteOutput());
838
+ remoteFileOutput.close();
839
+ }
840
+
841
+ private String defaultConfig(final String pathPrefix)
842
+ {
843
+ return "type: sftp\n" +
844
+ "host: " + HOST + "\n" +
845
+ "port: " + PORT + "\n" +
846
+ "user: " + USERNAME + "\n" +
847
+ "password: " + PASSWORD + "\n" +
848
+ "path_prefix: " + pathPrefix + "\n" +
849
+ "file_ext: txt\n" +
850
+ "formatter:\n" +
851
+ " type: csv\n" +
852
+ " newline: CRLF\n" +
853
+ " newline_in_field: LF\n" +
854
+ " header_line: true\n" +
855
+ " charset: UTF-8\n" +
856
+ " quote_policy: NONE\n" +
857
+ " quote: \"\\\"\"\n" +
858
+ " escape: \"\\\\\"\n" +
859
+ " null_string: \"\"\n" +
860
+ " default_timezone: 'UTC'";
861
+ }
862
+
863
+ private PluginTask defaultTask()
864
+ {
865
+ ConfigSource config = getConfigFromYaml(defaultConfig(defaultPathPrefix));
866
+ return config.loadConfig(SftpFileOutputPlugin.PluginTask.class);
867
+ }
868
+
869
+ private byte[] randBytes(final int len)
870
+ {
871
+ byte[] bytes = new byte[len];
872
+ new Random().nextBytes(bytes);
873
+ return bytes;
874
+ }
875
+
876
+ private File writeBytesToInputFile(final byte[] expected) throws IOException
877
+ {
878
+ File input = File.createTempFile("anything", ".dat");
879
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(input));
880
+ out.write(expected);
881
+ out.close();
882
+ return input;
883
+ }
884
+
885
+ private void assertEmptyUploadedFile(final String pathPrefix)
886
+ {
887
+ List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
888
+ assertThat(fileList, hasItem(containsString(pathPrefix)));
889
+
890
+ String filePath = testFolder.getRoot().getAbsolutePath() + File.separator + pathPrefix;
891
+ File output = new File(filePath);
892
+ assertEquals(0, output.length());
893
+ }
525
894
  }