embulk-output-sftp 0.1.11 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 98edf71b4a5d7998403671b7e181b250f7db9981
4
- data.tar.gz: 940f0d1fea8ed83c957bdfca08a2bb89725b487b
3
+ metadata.gz: 3140b0cc46176332801c67fbf286548785e5c5a3
4
+ data.tar.gz: a4fa818bcc4970eaf17e6a8b255ff06e35d605de
5
5
  SHA512:
6
- metadata.gz: c4e8426ac217df25329da8a514b1603158bc7d6ed2b630b2e28486fcd36fb81e24aee588e5b1b2f6dea2e0996789a947eaac617330cb6180ca3e6afc35a9f0ed
7
- data.tar.gz: 99c6f36a658a92dd3710c9b305892e5961f9d154694570b5f05e7cbf6e4ad2794adfda169e175622f3bb7ff5aa5714e25368e41a0f6224d3dce8acc004b87d9d
6
+ metadata.gz: e4a07a88cf20aebd20e14302d2f504809d509b630ed7c7dfc0e2de48d8d5f51d6b7172e8f9b86540a24dca1f1f2e4f32da92005b43f2a5fa88c3a5c55a390469
7
+ data.tar.gz: c71a56791aac8f6a68b6de286af446651e60719fe11996fa91d7a95ae356de87f1814dc068c84df7bb319d77262d6438f46d5e68163c1498a2c5dd62b7f461b8
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ 0.2.0 (2018-08-29)
2
+ - Fix bug of stalling when closing `remoteFile`, this version is recommended over 0.1.11
3
+ - https://github.com/embulk/embulk-output-sftp/pull/51
1
4
  0.1.11 (2018-08-27)
2
5
  - Enhance: Add 2 new configs `local_temp_file` (boolean) and `temp_file_threshold` (long)
3
6
  - https://github.com/embulk/embulk-output-sftp/pull/50
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  }
18
18
 
19
19
  group = "org.embulk.output.sftp"
20
- version = "0.1.11"
20
+ version = "0.2.0"
21
21
  sourceCompatibility = 1.7
22
22
  targetCompatibility = 1.7
23
23
 
@@ -20,6 +20,7 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
20
20
  import org.slf4j.Logger;
21
21
 
22
22
  import java.io.BufferedOutputStream;
23
+ import java.io.Closeable;
23
24
  import java.io.File;
24
25
  import java.io.FileInputStream;
25
26
  import java.io.IOException;
@@ -39,7 +40,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
39
40
  public class SftpUtils
40
41
  {
41
42
  private final Logger logger = Exec.getLogger(SftpUtils.class);
42
- private final DefaultFileSystemManager manager;
43
+ private DefaultFileSystemManager manager;
43
44
  private final FileSystemOptions fsOptions;
44
45
  private final String userInfo;
45
46
  private final String user;
@@ -159,15 +160,24 @@ public class SftpUtils
159
160
  final FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
160
161
  final BufferedOutputStream outputStream = openStream(remoteFile);
161
162
  // When channel is broken, closing resource may hang, hence the time-out wrapper
162
- // Note: closing FileObject will also close OutputStream
163
- try (TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
163
+ try (final TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
164
164
  appendFile(localTempFile, remoteFile, outputStream);
165
165
  return null;
166
166
  }
167
167
  finally {
168
- remoteFile.close();
168
+ // closing sequentially
169
+ new TimeoutCloser(remoteFile).close();
169
170
  }
170
171
  }
172
+
173
+ @Override
174
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
175
+ {
176
+ super.onRetry(exception, retryCount, retryLimit, retryWait);
177
+ // re-connect
178
+ manager.close();
179
+ manager = initializeStandardFileSystemManager();
180
+ }
171
181
  });
172
182
  }
173
183
 
@@ -258,18 +268,9 @@ public class SftpUtils
258
268
  return manager.resolveFile(getSftpFileUri(remoteFilePath).toString(), fsOptions);
259
269
  }
260
270
 
261
- BufferedOutputStream openStream(final FileObject remoteFile)
271
+ BufferedOutputStream openStream(final FileObject remoteFile) throws FileSystemException
262
272
  {
263
- // output stream is already a BufferedOutputStream, no need to wrap
264
- final String taskName = "SFTP open stream";
265
- return withRetry(new DefaultRetry<BufferedOutputStream>(taskName)
266
- {
267
- @Override
268
- public BufferedOutputStream call() throws Exception
269
- {
270
- return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
271
- }
272
- });
273
+ return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
273
274
  }
274
275
 
275
276
  URI getSftpFileUri(String remoteFilePath)
@@ -10,8 +10,7 @@ import java.util.concurrent.TimeoutException;
10
10
 
11
11
  public class TimeoutCloser implements Closeable
12
12
  {
13
- @VisibleForTesting
14
- int timeout = 300; // 5 minutes
13
+ private static int timeout = 300; // 5 minutes
15
14
  private Closeable wrapped;
16
15
 
17
16
  public TimeoutCloser(Closeable wrapped)
@@ -39,4 +38,10 @@ public class TimeoutCloser implements Closeable
39
38
  throw Throwables.propagate(e);
40
39
  }
41
40
  }
41
+
42
+ @VisibleForTesting
43
+ public static void setTimeout(int timeout)
44
+ {
45
+ TimeoutCloser.timeout = timeout;
46
+ }
42
47
  }
@@ -768,7 +768,7 @@ public class TestSftpFileOutputPlugin
768
768
  }
769
769
 
770
770
  @Test
771
- public void testOpenStreamWithRetry() throws FileSystemException
771
+ public void testOpenStreamWithoutRetry() throws FileSystemException
772
772
  {
773
773
  SftpFileOutputPlugin.PluginTask task = defaultTask();
774
774
  SftpUtils utils = new SftpUtils(task);
@@ -778,9 +778,13 @@ public class TestSftpFileOutputPlugin
778
778
  .doCallRealMethod()
779
779
  .when(mock).getContent();
780
780
 
781
- OutputStream stream = utils.openStream(mock);
782
- assertNotNull(stream);
783
- Mockito.verify(mock, Mockito.times(2)).getContent();
781
+ try {
782
+ utils.openStream(mock);
783
+ fail("Should not reach here");
784
+ }
785
+ catch (FileSystemException e) {
786
+ Mockito.verify(mock, Mockito.times(1)).getContent();
787
+ }
784
788
  }
785
789
 
786
790
  @Test
@@ -32,7 +32,7 @@ public class TestTimeoutCloser
32
32
  }
33
33
  }
34
34
  });
35
- closer.timeout = 1;
35
+ TimeoutCloser.setTimeout(1);
36
36
  try {
37
37
  closer.close();
38
38
  fail("Should not finish");
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-sftp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.11
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2018-08-27 00:00:00.000000000 Z
12
+ date: 2018-08-29 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -83,7 +83,7 @@ files:
83
83
  - src/test/resources/id_rsa.pub
84
84
  - classpath/commons-logging-1.2.jar
85
85
  - classpath/commons-vfs2-2.2.jar
86
- - classpath/embulk-output-sftp-0.1.11.jar
86
+ - classpath/embulk-output-sftp-0.2.0.jar
87
87
  - classpath/commons-io-2.6.jar
88
88
  - classpath/jsch-0.1.54.jar
89
89
  homepage: https://github.com/embulk/embulk-output-sftp