embulk-output-sftp 0.1.10 → 0.1.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,5 +1,6 @@
1
1
  package org.embulk.output.sftp;
2
2
 
3
+ import com.google.common.annotations.VisibleForTesting;
3
4
  import com.google.common.base.Function;
4
5
  import com.google.common.base.Throwables;
5
6
  import org.apache.commons.vfs2.FileObject;
@@ -9,6 +10,9 @@ import org.apache.commons.vfs2.impl.DefaultFileSystemManager;
9
10
  import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
10
11
  import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
11
12
  import org.embulk.config.ConfigException;
13
+ import org.embulk.output.sftp.utils.DefaultRetry;
14
+ import org.embulk.output.sftp.utils.TimedCallable;
15
+ import org.embulk.output.sftp.utils.TimeoutCloser;
12
16
  import org.embulk.spi.Exec;
13
17
  import org.embulk.spi.unit.LocalFile;
14
18
  import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
@@ -20,8 +24,10 @@ import java.io.File;
20
24
  import java.io.FileInputStream;
21
25
  import java.io.IOException;
22
26
  import java.io.InputStream;
27
+ import java.io.OutputStream;
23
28
  import java.net.URI;
24
29
  import java.net.URISyntaxException;
30
+ import java.util.concurrent.TimeUnit;
25
31
  import java.util.regex.Pattern;
26
32
 
27
33
  import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
@@ -40,6 +46,8 @@ public class SftpUtils
40
46
  private final String host;
41
47
  private final int port;
42
48
  private final int maxConnectionRetry;
49
+ @VisibleForTesting
50
+ int writeTimeout = 300; // 5 minutes
43
51
 
44
52
  private DefaultFileSystemManager initializeStandardFileSystemManager()
45
53
  {
@@ -53,7 +61,7 @@ public class SftpUtils
53
61
  * https://github.com/embulk/embulk-output-sftp/issues/40
54
62
  * https://github.com/embulk/embulk-output-sftp/pull/44
55
63
  * https://issues.apache.org/jira/browse/VFS-590
56
- */
64
+ */
57
65
  DefaultFileSystemManager manager = new DefaultFileSystemManager();
58
66
  try {
59
67
  manager.addProvider("sftp", new org.embulk.output.sftp.provider.sftp.SftpFileProvider());
@@ -87,8 +95,8 @@ public class SftpUtils
87
95
  builder.setStrictHostKeyChecking(fsOptions, "no");
88
96
  if (task.getSecretKeyFilePath().isPresent()) {
89
97
  IdentityInfo identityInfo = new IdentityInfo(
90
- new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
91
- task.getSecretKeyPassphrase().getBytes()
98
+ new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
99
+ task.getSecretKeyPassphrase().getBytes()
92
100
  );
93
101
  builder.setIdentityInfo(fsOptions, identityInfo);
94
102
  logger.info("set identity: {}", task.getSecretKeyFilePath().get());
@@ -141,132 +149,92 @@ public class SftpUtils
141
149
  manager.close();
142
150
  }
143
151
 
144
- public Void uploadFile(final File localTempFile, final String remotePath)
152
+ void uploadFile(final File localTempFile, final String remotePath)
145
153
  {
146
- try {
147
- return retryExecutor()
148
- .withRetryLimit(maxConnectionRetry)
149
- .withInitialRetryWait(500)
150
- .withMaxRetryWait(30 * 1000)
151
- .runInterruptible(new Retryable<Void>() {
152
- @Override
153
- public Void call() throws IOException
154
- {
155
- long size = localTempFile.length();
156
- int step = 10; // 10% each step
157
- long bytesPerStep = size / step;
158
- long startTime = System.nanoTime();
159
-
160
- try (FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
161
- InputStream inputStream = new FileInputStream(localTempFile);
162
- BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
163
- ) {
164
- logger.info("Uploading to remote sftp file ({} KB): {}", size / 1024, remoteFile.getPublicURIString());
165
- byte[] buffer = new byte[32 * 1024 * 1024]; // 32MB buffer size
166
- int len = inputStream.read(buffer);
167
- long total = 0;
168
- int progress = 0;
169
- while (len != -1) {
170
- outputStream.write(buffer, 0, len);
171
- len = inputStream.read(buffer);
172
- total += len;
173
- if (total / bytesPerStep > progress) {
174
- progress = (int) (total / bytesPerStep);
175
- long transferRate = (long) (total / ((System.nanoTime() - startTime) / 1e9));
176
- logger.info("Upload progress: {}% - {} KB - {} KB/s",
177
- progress * step, total / 1024, transferRate / 1024);
178
- }
179
- }
180
- logger.info("Upload completed.");
181
- }
182
- return null;
183
- }
184
-
185
- @Override
186
- public boolean isRetryableException(Exception exception)
187
- {
188
- if (exception instanceof ConfigException) {
189
- return false;
190
- }
191
- return true;
192
- }
193
-
194
- @Override
195
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
196
- {
197
- String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
198
- retryCount, retryLimit, retryWait / 1000, exception.getMessage());
199
- if (retryCount % 3 == 0) {
200
- logger.warn(message, exception);
201
- }
202
- else {
203
- logger.warn(message);
204
- }
205
- }
206
-
207
- @Override
208
- public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
209
- {
210
- }
211
- });
212
- }
213
- catch (RetryGiveupException ex) {
214
- throw Throwables.propagate(ex.getCause());
215
- }
216
- catch (InterruptedException ex) {
217
- throw Throwables.propagate(ex);
154
+ withRetry(new DefaultRetry<Void>(String.format("SFTP upload file '%s'", remotePath))
155
+ {
156
+ @Override
157
+ public Void call() throws Exception
158
+ {
159
+ final FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
160
+ final BufferedOutputStream outputStream = openStream(remoteFile);
161
+ // When channel is broken, closing resource may hang, hence the time-out wrapper
162
+ // Note: closing FileObject will also close OutputStream
163
+ try (TimeoutCloser ignored = new TimeoutCloser(outputStream)) {
164
+ appendFile(localTempFile, remoteFile, outputStream);
165
+ return null;
166
+ }
167
+ finally {
168
+ remoteFile.close();
169
+ }
170
+ }
171
+ });
172
+ }
173
+
174
+ /**
175
+ * This method won't close outputStream, outputStream is intended to keep open for next write
176
+ *
177
+ * @param localTempFile
178
+ * @param remoteFile
179
+ * @param outputStream
180
+ * @throws IOException
181
+ */
182
+ void appendFile(final File localTempFile, final FileObject remoteFile, final BufferedOutputStream outputStream) throws IOException
183
+ {
184
+ long size = localTempFile.length();
185
+ int step = 10; // 10% each step
186
+ long bytesPerStep = Math.max(size / step, 1); // to prevent / 0 if file size < 10 bytes
187
+ long startTime = System.nanoTime();
188
+
189
+ // start uploading
190
+ try (InputStream inputStream = new FileInputStream(localTempFile)) {
191
+ logger.info("Uploading to remote sftp file ({} KB): {}", size / 1024, remoteFile.getPublicURIString());
192
+ final byte[] buffer = new byte[32 * 1024 * 1024]; // 32MB buffer size
193
+ int len = inputStream.read(buffer);
194
+ long total = 0;
195
+ int progress = 0;
196
+ while (len != -1) {
197
+ timedWrite(outputStream, buffer, len);
198
+ len = inputStream.read(buffer);
199
+ total += len;
200
+ if (total / bytesPerStep > progress) {
201
+ progress = (int) (total / bytesPerStep);
202
+ long transferRate = (long) (total / ((System.nanoTime() - startTime) / 1e9));
203
+ logger.info("Upload progress: {}% - {} KB - {} KB/s",
204
+ progress * step, total / 1024, transferRate / 1024);
205
+ }
206
+ }
207
+ logger.info("Upload completed.");
218
208
  }
219
209
  }
220
210
 
221
211
  public Void renameFile(final String before, final String after)
212
+ {
213
+ return withRetry(new DefaultRetry<Void>("SFTP rename remote file")
214
+ {
215
+ @Override
216
+ public Void call() throws IOException
217
+ {
218
+ FileObject previousFile = resolve(before);
219
+ FileObject afterFile = resolve(after);
220
+ previousFile.moveTo(afterFile);
221
+ logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
222
+
223
+ return null;
224
+ }
225
+ });
226
+ }
227
+
228
+ public void deleteFile(final String remotePath)
222
229
  {
223
230
  try {
224
- return retryExecutor()
225
- .withRetryLimit(maxConnectionRetry)
226
- .withInitialRetryWait(500)
227
- .withMaxRetryWait(30 * 1000)
228
- .runInterruptible(new Retryable<Void>() {
229
- @Override
230
- public Void call() throws IOException
231
- {
232
- FileObject previousFile = manager.resolveFile(getSftpFileUri(before).toString(), fsOptions);
233
- FileObject afterFile = manager.resolveFile(getSftpFileUri(after).toString(), fsOptions);
234
- previousFile.moveTo(afterFile);
235
- logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
236
-
237
- return null;
238
- }
239
-
240
- @Override
241
- public boolean isRetryableException(Exception exception)
242
- {
243
- return true;
244
- }
245
-
246
- @Override
247
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
248
- {
249
- String message = String.format("SFTP rename remote file failed. Retrying %d/%d after %d seconds. Message: %s",
250
- retryCount, retryLimit, retryWait / 1000, exception.getMessage());
251
- if (retryCount % 3 == 0) {
252
- logger.warn(message, exception);
253
- }
254
- else {
255
- logger.warn(message);
256
- }
257
- }
258
-
259
- @Override
260
- public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
261
- {
262
- }
263
- });
264
- }
265
- catch (RetryGiveupException ex) {
266
- throw Throwables.propagate(ex.getCause());
231
+ FileObject file = manager.resolveFile(getSftpFileUri(remotePath).toString(), fsOptions);
232
+ if (file.exists()) {
233
+ file.delete();
234
+ }
267
235
  }
268
- catch (InterruptedException ex) {
269
- throw Throwables.propagate(ex);
236
+ catch (FileSystemException e) {
237
+ logger.warn("Failed to delete remote file '{}': {}", remotePath, e.getMessage());
270
238
  }
271
239
  }
272
240
 
@@ -285,7 +253,26 @@ public class SftpUtils
285
253
  }
286
254
  }
287
255
 
288
- private URI getSftpFileUri(String remoteFilePath)
256
+ FileObject resolve(final String remoteFilePath) throws FileSystemException
257
+ {
258
+ return manager.resolveFile(getSftpFileUri(remoteFilePath).toString(), fsOptions);
259
+ }
260
+
261
+ BufferedOutputStream openStream(final FileObject remoteFile)
262
+ {
263
+ // output stream is already a BufferedOutputStream, no need to wrap
264
+ final String taskName = "SFTP open stream";
265
+ return withRetry(new DefaultRetry<BufferedOutputStream>(taskName)
266
+ {
267
+ @Override
268
+ public BufferedOutputStream call() throws Exception
269
+ {
270
+ return new BufferedOutputStream(remoteFile.getContent().getOutputStream());
271
+ }
272
+ });
273
+ }
274
+
275
+ URI getSftpFileUri(String remoteFilePath)
289
276
  {
290
277
  try {
291
278
  return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
@@ -296,7 +283,7 @@ public class SftpUtils
296
283
  }
297
284
  }
298
285
 
299
- private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
286
+ FileObject newSftpFile(final URI sftpUri) throws FileSystemException
300
287
  {
301
288
  FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
302
289
  if (file.exists()) {
@@ -322,4 +309,40 @@ public class SftpUtils
322
309
  }
323
310
  };
324
311
  }
312
+
313
+ private <T> T withRetry(Retryable<T> call)
314
+ {
315
+ try {
316
+ return retryExecutor()
317
+ .withRetryLimit(maxConnectionRetry)
318
+ .withInitialRetryWait(500)
319
+ .withMaxRetryWait(30 * 1000)
320
+ .runInterruptible(call);
321
+ }
322
+ catch (RetryGiveupException ex) {
323
+ throw Throwables.propagate(ex.getCause());
324
+ }
325
+ catch (InterruptedException ex) {
326
+ throw Throwables.propagate(ex);
327
+ }
328
+ }
329
+
330
+ private void timedWrite(final OutputStream stream, final byte[] buf, final int len) throws IOException
331
+ {
332
+ try {
333
+ new TimedCallable<Void>()
334
+ {
335
+ @Override
336
+ public Void call() throws Exception
337
+ {
338
+ stream.write(buf, 0, len);
339
+ return null;
340
+ }
341
+ }.call(writeTimeout, TimeUnit.SECONDS);
342
+ }
343
+ catch (Exception e) {
344
+ logger.warn("Failed to write buffer, aborting ... ");
345
+ throw new IOException(e);
346
+ }
347
+ }
325
348
  }
@@ -0,0 +1,55 @@
1
+ package org.embulk.output.sftp.utils;
2
+
3
+ import com.jcraft.jsch.JSchException;
4
+ import org.embulk.spi.Exec;
5
+ import org.embulk.spi.util.RetryExecutor;
6
+ import org.slf4j.Logger;
7
+
8
+ public abstract class DefaultRetry<T> implements RetryExecutor.Retryable<T>
9
+ {
10
+ private Logger logger = Exec.getLogger(getClass());
11
+
12
+ private final String task;
13
+
14
+ protected DefaultRetry(String task)
15
+ {
16
+ this.task = task;
17
+ }
18
+
19
+ @Override
20
+ public boolean isRetryableException(Exception exception)
21
+ {
22
+ return !hasRootCauseAuthFail(exception);
23
+ }
24
+
25
+ @Override
26
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait)
27
+ {
28
+ String message = String.format("%s failed. Retrying %d/%d after %d seconds. Message: %s",
29
+ task, retryCount, retryLimit, retryWait / 1000, exception.getMessage());
30
+ if (retryCount % 3 == 0) {
31
+ logger.warn(message, exception);
32
+ }
33
+ else {
34
+ logger.warn(message);
35
+ }
36
+ }
37
+
38
+ @Override
39
+ public void onGiveup(Exception firstException, Exception lastException)
40
+ {
41
+ }
42
+
43
+ private static boolean isAuthFail(Throwable e)
44
+ {
45
+ return e instanceof JSchException && "USERAUTH fail".equals(e.getMessage());
46
+ }
47
+
48
+ private static boolean hasRootCauseAuthFail(Throwable e)
49
+ {
50
+ while (e != null && !isAuthFail(e)) {
51
+ e = e.getCause();
52
+ }
53
+ return e != null;
54
+ }
55
+ }
@@ -0,0 +1,27 @@
1
+ package org.embulk.output.sftp.utils;
2
+
3
+ import java.util.concurrent.Callable;
4
+ import java.util.concurrent.ExecutionException;
5
+ import java.util.concurrent.ExecutorService;
6
+ import java.util.concurrent.Executors;
7
+ import java.util.concurrent.FutureTask;
8
+ import java.util.concurrent.TimeUnit;
9
+ import java.util.concurrent.TimeoutException;
10
+
11
+ public abstract class TimedCallable<V> implements Callable<V>
12
+ {
13
+ private static final ExecutorService THREAD_POOL = Executors.newCachedThreadPool();
14
+
15
+ public V call(long timeout, TimeUnit timeUnit)
16
+ throws InterruptedException, ExecutionException, TimeoutException
17
+ {
18
+ FutureTask<V> task = new FutureTask<>(this);
19
+ try {
20
+ THREAD_POOL.execute(task);
21
+ return task.get(timeout, timeUnit);
22
+ }
23
+ finally {
24
+ task.cancel(true);
25
+ }
26
+ }
27
+ }
@@ -0,0 +1,42 @@
1
+ package org.embulk.output.sftp.utils;
2
+
3
+ import com.google.common.annotations.VisibleForTesting;
4
+ import com.google.common.base.Throwables;
5
+
6
+ import java.io.Closeable;
7
+ import java.util.concurrent.ExecutionException;
8
+ import java.util.concurrent.TimeUnit;
9
+ import java.util.concurrent.TimeoutException;
10
+
11
+ public class TimeoutCloser implements Closeable
12
+ {
13
+ @VisibleForTesting
14
+ int timeout = 300; // 5 minutes
15
+ private Closeable wrapped;
16
+
17
+ public TimeoutCloser(Closeable wrapped)
18
+ {
19
+ this.wrapped = wrapped;
20
+ }
21
+
22
+ @Override
23
+ public void close()
24
+ {
25
+ try {
26
+ new TimedCallable<Void>()
27
+ {
28
+ @Override
29
+ public Void call() throws Exception
30
+ {
31
+ if (wrapped != null) {
32
+ wrapped.close();
33
+ }
34
+ return null;
35
+ }
36
+ }.call(timeout, TimeUnit.SECONDS);
37
+ }
38
+ catch (InterruptedException | ExecutionException | TimeoutException e) {
39
+ throw Throwables.propagate(e);
40
+ }
41
+ }
42
+ }