embulk-output-sftp 0.1.11 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/build.gradle +1 -1
- data/classpath/{embulk-output-sftp-0.1.11.jar → embulk-output-sftp-0.2.0.jar} +0 -0
- data/src/main/java/org/embulk/output/sftp/SftpUtils.java +16 -15
- data/src/main/java/org/embulk/output/sftp/utils/TimeoutCloser.java +7 -2
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +8 -4
- data/src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3140b0cc46176332801c67fbf286548785e5c5a3
|
4
|
+
data.tar.gz: a4fa818bcc4970eaf17e6a8b255ff06e35d605de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e4a07a88cf20aebd20e14302d2f504809d509b630ed7c7dfc0e2de48d8d5f51d6b7172e8f9b86540a24dca1f1f2e4f32da92005b43f2a5fa88c3a5c55a390469
|
7
|
+
data.tar.gz: c71a56791aac8f6a68b6de286af446651e60719fe11996fa91d7a95ae356de87f1814dc068c84df7bb319d77262d6438f46d5e68163c1498a2c5dd62b7f461b8
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,6 @@
|
|
1
|
+
0.2.0 (2018-08-29)
|
2
|
+
- Fix bug of stalling when closing `remoteFile`, this version is recommended over 0.1.11
|
3
|
+
- https://github.com/embulk/embulk-output-sftp/pull/51
|
1
4
|
0.1.11 (2018-08-27)
|
2
5
|
- Enhance: Add 2 new configs `local_temp_file` (boolean) and `temp_file_threshold` (long)
|
3
6
|
- https://github.com/embulk/embulk-output-sftp/pull/50
|
data/build.gradle
CHANGED
Binary file
|
@@ -20,6 +20,7 @@ import org.embulk.spi.util.RetryExecutor.Retryable;
|
|
20
20
|
import org.slf4j.Logger;
|
21
21
|
|
22
22
|
import java.io.BufferedOutputStream;
|
23
|
+
import java.io.Closeable;
|
23
24
|
import java.io.File;
|
24
25
|
import java.io.FileInputStream;
|
25
26
|
import java.io.IOException;
|
@@ -39,7 +40,7 @@ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
|
39
40
|
public class SftpUtils
|
40
41
|
{
|
41
42
|
private final Logger logger = Exec.getLogger(SftpUtils.class);
|
42
|
-
private
|
43
|
+
private DefaultFileSystemManager manager;
|
43
44
|
private final FileSystemOptions fsOptions;
|
44
45
|
private final String userInfo;
|
45
46
|
private final String user;
|
@@ -159,15 +160,24 @@ public class SftpUtils
|
|
159
160
|
final FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
|
160
161
|
final BufferedOutputStream outputStream = openStream(remoteFile);
|
161
162
|
// When channel is broken, closing resource may hang, hence the time-out wrapper
|
162
|
-
|
163
|
-
try (TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
|
163
|
+
try (final TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
|
164
164
|
appendFile(localTempFile, remoteFile, outputStream);
|
165
165
|
return null;
|
166
166
|
}
|
167
167
|
finally {
|
168
|
-
|
168
|
+
// closing sequentially
|
169
|
+
new TimeoutCloser(remoteFile).close();
|
169
170
|
}
|
170
171
|
}
|
172
|
+
|
173
|
+
@Override
|
174
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
175
|
+
{
|
176
|
+
super.onRetry(exception, retryCount, retryLimit, retryWait);
|
177
|
+
// re-connect
|
178
|
+
manager.close();
|
179
|
+
manager = initializeStandardFileSystemManager();
|
180
|
+
}
|
171
181
|
});
|
172
182
|
}
|
173
183
|
|
@@ -258,18 +268,9 @@ public class SftpUtils
|
|
258
268
|
return manager.resolveFile(getSftpFileUri(remoteFilePath).toString(), fsOptions);
|
259
269
|
}
|
260
270
|
|
261
|
-
BufferedOutputStream openStream(final FileObject remoteFile)
|
271
|
+
BufferedOutputStream openStream(final FileObject remoteFile) throws FileSystemException
|
262
272
|
{
|
263
|
-
|
264
|
-
final String taskName = "SFTP open stream";
|
265
|
-
return withRetry(new DefaultRetry<BufferedOutputStream>(taskName)
|
266
|
-
{
|
267
|
-
@Override
|
268
|
-
public BufferedOutputStream call() throws Exception
|
269
|
-
{
|
270
|
-
return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
|
271
|
-
}
|
272
|
-
});
|
273
|
+
return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
|
273
274
|
}
|
274
275
|
|
275
276
|
URI getSftpFileUri(String remoteFilePath)
|
@@ -10,8 +10,7 @@ import java.util.concurrent.TimeoutException;
|
|
10
10
|
|
11
11
|
public class TimeoutCloser implements Closeable
|
12
12
|
{
|
13
|
-
|
14
|
-
int timeout = 300; // 5 minutes
|
13
|
+
private static int timeout = 300; // 5 minutes
|
15
14
|
private Closeable wrapped;
|
16
15
|
|
17
16
|
public TimeoutCloser(Closeable wrapped)
|
@@ -39,4 +38,10 @@ public class TimeoutCloser implements Closeable
|
|
39
38
|
throw Throwables.propagate(e);
|
40
39
|
}
|
41
40
|
}
|
41
|
+
|
42
|
+
@VisibleForTesting
|
43
|
+
public static void setTimeout(int timeout)
|
44
|
+
{
|
45
|
+
TimeoutCloser.timeout = timeout;
|
46
|
+
}
|
42
47
|
}
|
@@ -768,7 +768,7 @@ public class TestSftpFileOutputPlugin
|
|
768
768
|
}
|
769
769
|
|
770
770
|
@Test
|
771
|
-
public void
|
771
|
+
public void testOpenStreamWithoutRetry() throws FileSystemException
|
772
772
|
{
|
773
773
|
SftpFileOutputPlugin.PluginTask task = defaultTask();
|
774
774
|
SftpUtils utils = new SftpUtils(task);
|
@@ -778,9 +778,13 @@ public class TestSftpFileOutputPlugin
|
|
778
778
|
.doCallRealMethod()
|
779
779
|
.when(mock).getContent();
|
780
780
|
|
781
|
-
|
782
|
-
|
783
|
-
|
781
|
+
try {
|
782
|
+
utils.openStream(mock);
|
783
|
+
fail("Should not reach here");
|
784
|
+
}
|
785
|
+
catch (FileSystemException e) {
|
786
|
+
Mockito.verify(mock, Mockito.times(1)).getContent();
|
787
|
+
}
|
784
788
|
}
|
785
789
|
|
786
790
|
@Test
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-sftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-08-
|
12
|
+
date: 2018-08-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -83,7 +83,7 @@ files:
|
|
83
83
|
- src/test/resources/id_rsa.pub
|
84
84
|
- classpath/commons-logging-1.2.jar
|
85
85
|
- classpath/commons-vfs2-2.2.jar
|
86
|
-
- classpath/embulk-output-sftp-0.
|
86
|
+
- classpath/embulk-output-sftp-0.2.0.jar
|
87
87
|
- classpath/commons-io-2.6.jar
|
88
88
|
- classpath/jsch-0.1.54.jar
|
89
89
|
homepage: https://github.com/embulk/embulk-output-sftp
|