embulk-output-sftp 0.1.10 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +2 -3
- data/CHANGELOG.md +3 -0
- data/README.md +3 -0
- data/build.gradle +6 -5
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +20 -3
- data/src/main/java/org/embulk/output/sftp/SftpLocalFileOutput.java +223 -0
- data/src/main/java/org/embulk/output/sftp/SftpRemoteFileOutput.java +120 -0
- data/src/main/java/org/embulk/output/sftp/SftpUtils.java +146 -123
- data/src/main/java/org/embulk/output/sftp/utils/DefaultRetry.java +55 -0
- data/src/main/java/org/embulk/output/sftp/utils/TimedCallable.java +27 -0
- data/src/main/java/org/embulk/output/sftp/utils/TimeoutCloser.java +42 -0
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +377 -8
- data/src/test/java/org/embulk/output/sftp/utils/TestTimedCallable.java +36 -0
- data/src/test/java/org/embulk/output/sftp/utils/TestTimeoutCloser.java +45 -0
- metadata +11 -5
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +0 -140
@@ -1,5 +1,6 @@
|
|
1
1
|
package org.embulk.output.sftp;
|
2
2
|
|
3
|
+
import com.google.common.annotations.VisibleForTesting;
|
3
4
|
import com.google.common.base.Function;
|
4
5
|
import com.google.common.base.Throwables;
|
5
6
|
import org.apache.commons.vfs2.FileObject;
|
@@ -9,6 +10,9 @@ import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
|
|
9
10
|
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
10
11
|
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
11
12
|
import org.embulk.config.ConfigException;
|
13
|
+
import org.embulk.output.sftp.utils.DefaultRetry;
|
14
|
+
import org.embulk.output.sftp.utils.TimedCallable;
|
15
|
+
import org.embulk.output.sftp.utils.TimeoutCloser;
|
12
16
|
import org.embulk.spi.Exec;
|
13
17
|
import org.embulk.spi.unit.LocalFile;
|
14
18
|
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
@@ -20,8 +24,10 @@ import java.io.File;
|
|
20
24
|
import java.io.FileInputStream;
|
21
25
|
import java.io.IOException;
|
22
26
|
import java.io.InputStream;
|
27
|
+
import java.io.OutputStream;
|
23
28
|
import java.net.URI;
|
24
29
|
import java.net.URISyntaxException;
|
30
|
+
import java.util.concurrent.TimeUnit;
|
25
31
|
import java.util.regex.Pattern;
|
26
32
|
|
27
33
|
import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
|
@@ -40,6 +46,8 @@ public class SftpUtils
|
|
40
46
|
private final String host;
|
41
47
|
private final int port;
|
42
48
|
private final int maxConnectionRetry;
|
49
|
+
@VisibleForTesting
|
50
|
+
int writeTimeout = 300; // 5 minutes
|
43
51
|
|
44
52
|
private DefaultFileSystemManager initializeStandardFileSystemManager()
|
45
53
|
{
|
@@ -53,7 +61,7 @@ public class SftpUtils
|
|
53
61
|
* https://github.com/embulk/embulk-output-sftp/issues/40
|
54
62
|
* https://github.com/embulk/embulk-output-sftp/pull/44
|
55
63
|
* https://issues.apache.org/jira/browse/VFS-590
|
56
|
-
|
64
|
+
*/
|
57
65
|
DefaultFileSystemManager manager = new DefaultFileSystemManager();
|
58
66
|
try {
|
59
67
|
manager.addProvider("sftp", new org.embulk.output.sftp.provider.sftp.SftpFileProvider());
|
@@ -87,8 +95,8 @@ public class SftpUtils
|
|
87
95
|
builder.setStrictHostKeyChecking(fsOptions, "no");
|
88
96
|
if (task.getSecretKeyFilePath().isPresent()) {
|
89
97
|
IdentityInfo identityInfo = new IdentityInfo(
|
90
|
-
|
91
|
-
|
98
|
+
new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
|
99
|
+
task.getSecretKeyPassphrase().getBytes()
|
92
100
|
);
|
93
101
|
builder.setIdentityInfo(fsOptions, identityInfo);
|
94
102
|
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
@@ -141,132 +149,92 @@ public class SftpUtils
|
|
141
149
|
manager.close();
|
142
150
|
}
|
143
151
|
|
144
|
-
|
152
|
+
void uploadFile(final File localTempFile, final String remotePath)
|
145
153
|
{
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
logger.warn(message, exception);
|
201
|
-
}
|
202
|
-
else {
|
203
|
-
logger.warn(message);
|
204
|
-
}
|
205
|
-
}
|
206
|
-
|
207
|
-
@Override
|
208
|
-
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
209
|
-
{
|
210
|
-
}
|
211
|
-
});
|
212
|
-
}
|
213
|
-
catch (RetryGiveupException ex) {
|
214
|
-
throw Throwables.propagate(ex.getCause());
|
215
|
-
}
|
216
|
-
catch (InterruptedException ex) {
|
217
|
-
throw Throwables.propagate(ex);
|
154
|
+
withRetry(new DefaultRetry<Void>(String.format("SFTP upload file '%s'", remotePath))
|
155
|
+
{
|
156
|
+
@Override
|
157
|
+
public Void call() throws Exception
|
158
|
+
{
|
159
|
+
final FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
|
160
|
+
final BufferedOutputStream outputStream = openStream(remoteFile);
|
161
|
+
// When channel is broken, closing resource may hang, hence the time-out wrapper
|
162
|
+
// Note: closing FileObject will also close OutputStream
|
163
|
+
try (TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
|
164
|
+
appendFile(localTempFile, remoteFile, outputStream);
|
165
|
+
return null;
|
166
|
+
}
|
167
|
+
finally {
|
168
|
+
remoteFile.close();
|
169
|
+
}
|
170
|
+
}
|
171
|
+
});
|
172
|
+
}
|
173
|
+
|
174
|
+
/**
|
175
|
+
* This method won't close outputStream, outputStream is intended to keep open for next write
|
176
|
+
*
|
177
|
+
* @param localTempFile
|
178
|
+
* @param remoteFile
|
179
|
+
* @param outputStream
|
180
|
+
* @throws IOException
|
181
|
+
*/
|
182
|
+
void appendFile(final File localTempFile, final FileObject remoteFile, final BufferedOutputStream outputStream) throws IOException
|
183
|
+
{
|
184
|
+
long size = localTempFile.length();
|
185
|
+
int step = 10; // 10% each step
|
186
|
+
long bytesPerStep = Math.max(size / step, 1); // to prevent / 0 if file size < 10 bytes
|
187
|
+
long startTime = System.nanoTime();
|
188
|
+
|
189
|
+
// start uploading
|
190
|
+
try (InputStream inputStream = new FileInputStream(localTempFile)) {
|
191
|
+
logger.info("Uploading to remote sftp file ({} KB): {}", size / 1024, remoteFile.getPublicURIString());
|
192
|
+
final byte[] buffer = new byte[32 * 1024 * 1024]; // 32MB buffer size
|
193
|
+
int len = inputStream.read(buffer);
|
194
|
+
long total = 0;
|
195
|
+
int progress = 0;
|
196
|
+
while (len != -1) {
|
197
|
+
timedWrite(outputStream, buffer, len);
|
198
|
+
len = inputStream.read(buffer);
|
199
|
+
total += len;
|
200
|
+
if (total / bytesPerStep > progress) {
|
201
|
+
progress = (int) (total / bytesPerStep);
|
202
|
+
long transferRate = (long) (total / ((System.nanoTime() - startTime) / 1e9));
|
203
|
+
logger.info("Upload progress: {}% - {} KB - {} KB/s",
|
204
|
+
progress * step, total / 1024, transferRate / 1024);
|
205
|
+
}
|
206
|
+
}
|
207
|
+
logger.info("Upload completed.");
|
218
208
|
}
|
219
209
|
}
|
220
210
|
|
221
211
|
public Void renameFile(final String before, final String after)
|
212
|
+
{
|
213
|
+
return withRetry(new DefaultRetry<Void>("SFTP rename remote file")
|
214
|
+
{
|
215
|
+
@Override
|
216
|
+
public Void call() throws IOException
|
217
|
+
{
|
218
|
+
FileObject previousFile = resolve(before);
|
219
|
+
FileObject afterFile = resolve(after);
|
220
|
+
previousFile.moveTo(afterFile);
|
221
|
+
logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
|
222
|
+
|
223
|
+
return null;
|
224
|
+
}
|
225
|
+
});
|
226
|
+
}
|
227
|
+
|
228
|
+
public void deleteFile(final String remotePath)
|
222
229
|
{
|
223
230
|
try {
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
.runInterruptible(new Retryable<Void>() {
|
229
|
-
@Override
|
230
|
-
public Void call() throws IOException
|
231
|
-
{
|
232
|
-
FileObject previousFile = manager.resolveFile(getSftpFileUri(before).toString(), fsOptions);
|
233
|
-
FileObject afterFile = manager.resolveFile(getSftpFileUri(after).toString(), fsOptions);
|
234
|
-
previousFile.moveTo(afterFile);
|
235
|
-
logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
|
236
|
-
|
237
|
-
return null;
|
238
|
-
}
|
239
|
-
|
240
|
-
@Override
|
241
|
-
public boolean isRetryableException(Exception exception)
|
242
|
-
{
|
243
|
-
return true;
|
244
|
-
}
|
245
|
-
|
246
|
-
@Override
|
247
|
-
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
|
248
|
-
{
|
249
|
-
String message = String.format("SFTP rename remote file failed. Retrying %d/%d after %d seconds. Message: %s",
|
250
|
-
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
251
|
-
if (retryCount % 3 == 0) {
|
252
|
-
logger.warn(message, exception);
|
253
|
-
}
|
254
|
-
else {
|
255
|
-
logger.warn(message);
|
256
|
-
}
|
257
|
-
}
|
258
|
-
|
259
|
-
@Override
|
260
|
-
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
261
|
-
{
|
262
|
-
}
|
263
|
-
});
|
264
|
-
}
|
265
|
-
catch (RetryGiveupException ex) {
|
266
|
-
throw Throwables.propagate(ex.getCause());
|
231
|
+
FileObject file = manager.resolveFile(getSftpFileUri(remotePath).toString(), fsOptions);
|
232
|
+
if (file.exists()) {
|
233
|
+
file.delete();
|
234
|
+
}
|
267
235
|
}
|
268
|
-
catch (
|
269
|
-
|
236
|
+
catch (FileSystemException e) {
|
237
|
+
logger.warn("Failed to delete remote file '{}': {}", remotePath, e.getMessage());
|
270
238
|
}
|
271
239
|
}
|
272
240
|
|
@@ -285,7 +253,26 @@ public class SftpUtils
|
|
285
253
|
}
|
286
254
|
}
|
287
255
|
|
288
|
-
|
256
|
+
FileObject resolve(final String remoteFilePath) throws FileSystemException
|
257
|
+
{
|
258
|
+
return manager.resolveFile(getSftpFileUri(remoteFilePath).toString(), fsOptions);
|
259
|
+
}
|
260
|
+
|
261
|
+
BufferedOutputStream openStream(final FileObject remoteFile)
|
262
|
+
{
|
263
|
+
// output stream is already a BufferedOutputStream, no need to wrap
|
264
|
+
final String taskName = "SFTP open stream";
|
265
|
+
return withRetry(new DefaultRetry<BufferedOutputStream>(taskName)
|
266
|
+
{
|
267
|
+
@Override
|
268
|
+
public BufferedOutputStream call() throws Exception
|
269
|
+
{
|
270
|
+
return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
|
271
|
+
}
|
272
|
+
});
|
273
|
+
}
|
274
|
+
|
275
|
+
URI getSftpFileUri(String remoteFilePath)
|
289
276
|
{
|
290
277
|
try {
|
291
278
|
return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
|
@@ -296,7 +283,7 @@ public class SftpUtils
|
|
296
283
|
}
|
297
284
|
}
|
298
285
|
|
299
|
-
|
286
|
+
FileObject newSftpFile(final URI sftpUri) throws FileSystemException
|
300
287
|
{
|
301
288
|
FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
|
302
289
|
if (file.exists()) {
|
@@ -322,4 +309,40 @@ public class SftpUtils
|
|
322
309
|
}
|
323
310
|
};
|
324
311
|
}
|
312
|
+
|
313
|
+
private <T> T withRetry(Retryable<T> call)
|
314
|
+
{
|
315
|
+
try {
|
316
|
+
return retryExecutor()
|
317
|
+
.withRetryLimit(maxConnectionRetry)
|
318
|
+
.withInitialRetryWait(500)
|
319
|
+
.withMaxRetryWait(30 * 1000)
|
320
|
+
.runInterruptible(call);
|
321
|
+
}
|
322
|
+
catch (RetryGiveupException ex) {
|
323
|
+
throw Throwables.propagate(ex.getCause());
|
324
|
+
}
|
325
|
+
catch (InterruptedException ex) {
|
326
|
+
throw Throwables.propagate(ex);
|
327
|
+
}
|
328
|
+
}
|
329
|
+
|
330
|
+
private void timedWrite(final OutputStream stream, final byte[] buf, final int len) throws IOException
|
331
|
+
{
|
332
|
+
try {
|
333
|
+
new TimedCallable<Void>()
|
334
|
+
{
|
335
|
+
@Override
|
336
|
+
public Void call() throws Exception
|
337
|
+
{
|
338
|
+
stream.write(buf, 0, len);
|
339
|
+
return null;
|
340
|
+
}
|
341
|
+
}.call(writeTimeout, TimeUnit.SECONDS);
|
342
|
+
}
|
343
|
+
catch (Exception e) {
|
344
|
+
logger.warn("Failed to write buffer, aborting ... ");
|
345
|
+
throw new IOException(e);
|
346
|
+
}
|
347
|
+
}
|
325
348
|
}
|
@@ -0,0 +1,55 @@
|
|
1
|
+
package org.embulk.output.sftp.utils;
|
2
|
+
|
3
|
+
import com.jcraft.jsch.JSchException;
|
4
|
+
import org.embulk.spi.Exec;
|
5
|
+
import org.embulk.spi.util.RetryExecutor;
|
6
|
+
import org.slf4j.Logger;
|
7
|
+
|
8
|
+
public abstract class DefaultRetry<T> implements RetryExecutor.Retryable<T>
|
9
|
+
{
|
10
|
+
private Logger logger = Exec.getLogger(getClass());
|
11
|
+
|
12
|
+
private final String task;
|
13
|
+
|
14
|
+
protected DefaultRetry(String task)
|
15
|
+
{
|
16
|
+
this.task = task;
|
17
|
+
}
|
18
|
+
|
19
|
+
@Override
|
20
|
+
public boolean isRetryableException(Exception exception)
|
21
|
+
{
|
22
|
+
return !hasRootCauseAuthFail(exception);
|
23
|
+
}
|
24
|
+
|
25
|
+
@Override
|
26
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
|
27
|
+
{
|
28
|
+
String message = String.format("%s failed. Retrying %d/%d after %d seconds. Message: %s",
|
29
|
+
task, retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
30
|
+
if (retryCount % 3 == 0) {
|
31
|
+
logger.warn(message, exception);
|
32
|
+
}
|
33
|
+
else {
|
34
|
+
logger.warn(message);
|
35
|
+
}
|
36
|
+
}
|
37
|
+
|
38
|
+
@Override
|
39
|
+
public void onGiveup(Exception firstException, Exception lastException)
|
40
|
+
{
|
41
|
+
}
|
42
|
+
|
43
|
+
private static boolean isAuthFail(Throwable e)
|
44
|
+
{
|
45
|
+
return e instanceof JSchException && "USERAUTH fail".equals(e.getMessage());
|
46
|
+
}
|
47
|
+
|
48
|
+
private static boolean hasRootCauseAuthFail(Throwable e)
|
49
|
+
{
|
50
|
+
while (e != null && !isAuthFail(e)) {
|
51
|
+
e = e.getCause();
|
52
|
+
}
|
53
|
+
return e != null;
|
54
|
+
}
|
55
|
+
}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
package org.embulk.output.sftp.utils;
|
2
|
+
|
3
|
+
import java.util.concurrent.Callable;
|
4
|
+
import java.util.concurrent.ExecutionException;
|
5
|
+
import java.util.concurrent.ExecutorService;
|
6
|
+
import java.util.concurrent.Executors;
|
7
|
+
import java.util.concurrent.FutureTask;
|
8
|
+
import java.util.concurrent.TimeUnit;
|
9
|
+
import java.util.concurrent.TimeoutException;
|
10
|
+
|
11
|
+
public abstract class TimedCallable<V> implements Callable<V>
|
12
|
+
{
|
13
|
+
private static final ExecutorService THREAD_POOL = Executors.newCachedThreadPool();
|
14
|
+
|
15
|
+
public V call(long timeout, TimeUnit timeUnit)
|
16
|
+
throws InterruptedException, ExecutionException, TimeoutException
|
17
|
+
{
|
18
|
+
FutureTask<V> task = new FutureTask<>(this);
|
19
|
+
try {
|
20
|
+
THREAD_POOL.execute(task);
|
21
|
+
return task.get(timeout, timeUnit);
|
22
|
+
}
|
23
|
+
finally {
|
24
|
+
task.cancel(true);
|
25
|
+
}
|
26
|
+
}
|
27
|
+
}
|
@@ -0,0 +1,42 @@
|
|
1
|
+
package org.embulk.output.sftp.utils;
|
2
|
+
|
3
|
+
import com.google.common.annotations.VisibleForTesting;
|
4
|
+
import com.google.common.base.Throwables;
|
5
|
+
|
6
|
+
import java.io.Closeable;
|
7
|
+
import java.util.concurrent.ExecutionException;
|
8
|
+
import java.util.concurrent.TimeUnit;
|
9
|
+
import java.util.concurrent.TimeoutException;
|
10
|
+
|
11
|
+
public class TimeoutCloser implements Closeable
|
12
|
+
{
|
13
|
+
@VisibleForTesting
|
14
|
+
int timeout = 300; // 5 minutes
|
15
|
+
private Closeable wrapped;
|
16
|
+
|
17
|
+
public TimeoutCloser(Closeable wrapped)
|
18
|
+
{
|
19
|
+
this.wrapped = wrapped;
|
20
|
+
}
|
21
|
+
|
22
|
+
@Override
|
23
|
+
public void close()
|
24
|
+
{
|
25
|
+
try {
|
26
|
+
new TimedCallable<Void>()
|
27
|
+
{
|
28
|
+
@Override
|
29
|
+
public Void call() throws Exception
|
30
|
+
{
|
31
|
+
if (wrapped != null) {
|
32
|
+
wrapped.close();
|
33
|
+
}
|
34
|
+
return null;
|
35
|
+
}
|
36
|
+
}.call(timeout, TimeUnit.SECONDS);
|
37
|
+
}
|
38
|
+
catch (InterruptedException | ExecutionException | TimeoutException e) {
|
39
|
+
throw Throwables.propagate(e);
|
40
|
+
}
|
41
|
+
}
|
42
|
+
}
|