embulk-output-sftp 0.1.2 → 0.1.3

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0c303670ed86cfbbf0185ef25f38bfaffe9034b9
4
- data.tar.gz: f8fcdabd44af66bc695f7d9134f14c41882430b7
3
+ metadata.gz: c59819e8adf5918b012907c1649584986ca9f77b
4
+ data.tar.gz: b942eb8bcaf5a55f18feae4ae68a6c0d4ef96b92
5
5
  SHA512:
6
- metadata.gz: 26531cc33e7c923bea492d3a7c3df607780668780f6c1897491f3c3eef5215844a8a145ee62ab3119ac237382cfb82783c3654a69fbb799af704f65c8653be0b
7
- data.tar.gz: 3bf1a4df8aa6c2b6920ed47200a708b578803a3984301e4a21c66056225415bc55a30dd99da48d496eda5d49efd54d827d454980909a421c2fd726fbf7d7100f
6
+ metadata.gz: 0abe7dba22a634303bc872806d348c2f2f380257c4ebf7cdd5de3e10bf56644e37e82e5ce72e49f8ac2c76169a77781c5aefdf96112f8bb28da37db8178cf97e
7
+ data.tar.gz: 4e4073715b98d80bf66fd1c24f846079a4faf73993ec7e619355309c2728bac6459183d053c45fcbb80d2cfbccc527ae466324f264d8260b1a52704e227603af
data/.travis.yml CHANGED
@@ -1,3 +1,4 @@
1
+ dist: precise
1
2
  language: java
2
3
  jdk:
3
4
  - openjdk7
@@ -6,4 +7,8 @@ jdk:
6
7
  script:
7
8
  - ./gradlew test
8
9
  after_success:
9
- - ./gradlew jacocoTestReport coveralls
10
+ - ./gradlew jacocoTestReport coveralls
11
+ addons:
12
+ hosts:
13
+ - example.com
14
+ hostname: example.com
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ 0.1.3 (2017-11-07)
2
+ - Enhance: Create temporary file and rename it after upload completed
3
+ - https://github.com/embulk/embulk-output-sftp/pull/37
4
+
1
5
  0.1.2 (2017-07-11)
2
6
  - Fix: Increment fileIndex after complete uploadFile
3
7
  - https://github.com/embulk/embulk-output-sftp/pull/35
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  }
18
18
 
19
19
  group = "org.embulk.output.sftp"
20
- version = "0.1.2"
20
+ version = "0.1.3"
21
21
  sourceCompatibility = 1.7
22
22
  targetCompatibility = 1.7
23
23
 
Binary file
@@ -1,6 +1,6 @@
1
- #Wed Jan 13 12:41:02 JST 2016
1
+ #Sun Jan 08 00:35:58 PST 2017
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
@@ -1,37 +1,24 @@
1
1
  package org.embulk.output.sftp;
2
2
 
3
- import com.google.common.base.Function;
4
3
  import com.google.common.base.Throwables;
5
- import org.apache.commons.io.IOUtils;
6
- import org.apache.commons.vfs2.FileObject;
7
- import org.apache.commons.vfs2.FileSystemException;
8
- import org.apache.commons.vfs2.FileSystemOptions;
9
- import org.apache.commons.vfs2.impl.StandardFileSystemManager;
10
- import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
11
- import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
12
- import org.embulk.config.ConfigException;
13
4
  import org.embulk.config.TaskReport;
14
5
  import org.embulk.spi.Buffer;
15
6
  import org.embulk.spi.Exec;
16
7
  import org.embulk.spi.FileOutput;
17
8
  import org.embulk.spi.TransactionalFileOutput;
18
- import org.embulk.spi.unit.LocalFile;
19
- import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
20
- import org.embulk.spi.util.RetryExecutor.Retryable;
21
9
  import org.slf4j.Logger;
22
10
 
23
- import java.io.BufferedInputStream;
24
11
  import java.io.BufferedOutputStream;
25
12
  import java.io.File;
26
- import java.io.FileInputStream;
27
13
  import java.io.FileNotFoundException;
28
14
  import java.io.FileOutputStream;
29
15
  import java.io.IOException;
30
- import java.net.URI;
31
- import java.net.URISyntaxException;
16
+ import java.util.ArrayList;
17
+ import java.util.HashMap;
18
+ import java.util.List;
19
+ import java.util.Map;
32
20
 
33
21
  import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
34
- import static org.embulk.spi.util.RetryExecutor.retryExecutor;
35
22
 
36
23
  /**
37
24
  * Created by takahiro.nakayama on 10/20/15.
@@ -40,110 +27,26 @@ public class SftpFileOutput
40
27
  implements FileOutput, TransactionalFileOutput
41
28
  {
42
29
  private final Logger logger = Exec.getLogger(SftpFileOutput.class);
43
- private final StandardFileSystemManager manager;
44
- private final FileSystemOptions fsOptions;
45
- private final String userInfo;
46
- private final String host;
47
- private final int port;
48
- private final int maxConnectionRetry;
49
30
  private final String pathPrefix;
50
31
  private final String sequenceFormat;
51
32
  private final String fileNameExtension;
52
33
 
53
34
  private final int taskIndex;
35
+ private final SftpUtils sftpUtils;
54
36
  private int fileIndex = 0;
55
37
  private File tempFile;
56
38
  private BufferedOutputStream localOutput = null;
39
+ private List<Map<String, String>> fileList = new ArrayList<>();
57
40
 
58
- private StandardFileSystemManager initializeStandardFileSystemManager()
59
- {
60
- if (!logger.isDebugEnabled()) {
61
- // TODO: change logging format: org.apache.commons.logging.Log
62
- System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
63
- }
64
- StandardFileSystemManager manager = new StandardFileSystemManager();
65
- manager.setClassLoader(SftpFileOutput.class.getClassLoader());
66
- try {
67
- manager.init();
68
- }
69
- catch (FileSystemException e) {
70
- logger.error(e.getMessage());
71
- throw new ConfigException(e);
72
- }
73
-
74
- return manager;
75
- }
76
-
77
- private String initializeUserInfo(PluginTask task)
78
- {
79
- String userInfo = task.getUser();
80
- if (task.getPassword().isPresent()) {
81
- userInfo += ":" + task.getPassword().get();
82
- }
83
- return userInfo;
84
- }
85
-
86
- private FileSystemOptions initializeFsOptions(PluginTask task)
87
- {
88
- FileSystemOptions fsOptions = new FileSystemOptions();
89
-
90
- try {
91
- SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
92
- builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
93
- builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
94
- builder.setStrictHostKeyChecking(fsOptions, "no");
95
- if (task.getSecretKeyFilePath().isPresent()) {
96
- IdentityInfo identityInfo = new IdentityInfo(
97
- new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
98
- task.getSecretKeyPassphrase().getBytes()
99
- );
100
- builder.setIdentityInfo(fsOptions, identityInfo);
101
- logger.info("set identity: {}", task.getSecretKeyFilePath().get());
102
- }
103
-
104
- if (task.getProxy().isPresent()) {
105
- ProxyTask proxy = task.getProxy().get();
106
-
107
- ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
108
-
109
- if (proxy.getHost().isPresent()) {
110
- builder.setProxyHost(fsOptions, proxy.getHost().get());
111
- builder.setProxyPort(fsOptions, proxy.getPort());
112
- }
113
-
114
- if (proxy.getUser().isPresent()) {
115
- builder.setProxyUser(fsOptions, proxy.getUser().get());
116
- }
117
-
118
- if (proxy.getPassword().isPresent()) {
119
- builder.setProxyPassword(fsOptions, proxy.getPassword().get());
120
- }
121
-
122
- if (proxy.getCommand().isPresent()) {
123
- builder.setProxyCommand(fsOptions, proxy.getCommand().get());
124
- }
125
- }
126
- }
127
- catch (FileSystemException e) {
128
- logger.error(e.getMessage());
129
- throw new ConfigException(e);
130
- }
131
-
132
- return fsOptions;
133
- }
41
+ private final String temporaryFileSuffix = ".tmp";
134
42
 
135
43
  SftpFileOutput(PluginTask task, int taskIndex)
136
44
  {
137
- this.manager = initializeStandardFileSystemManager();
138
- this.userInfo = initializeUserInfo(task);
139
- this.fsOptions = initializeFsOptions(task);
140
- this.host = task.getHost();
141
- this.port = task.getPort();
142
- this.maxConnectionRetry = task.getMaxConnectionRetry();
143
45
  this.pathPrefix = task.getPathPrefix();
144
46
  this.sequenceFormat = task.getSequenceFormat();
145
47
  this.fileNameExtension = task.getFileNameExtension();
146
48
  this.taskIndex = taskIndex;
49
+ this.sftpUtils = new SftpUtils(task);
147
50
  }
148
51
 
149
52
  @Override
@@ -179,7 +82,14 @@ public class SftpFileOutput
179
82
  public void finish()
180
83
  {
181
84
  closeCurrentFile();
182
- uploadFile(getOutputFilePath());
85
+ String fileName = getOutputFilePath();
86
+ String temporaryFileName = fileName + temporaryFileSuffix;
87
+ sftpUtils.uploadFile(tempFile, temporaryFileName);
88
+
89
+ Map<String, String> executedFiles = new HashMap<>();
90
+ executedFiles.put("temporary_filename", temporaryFileName);
91
+ executedFiles.put("real_filename", fileName);
92
+ fileList.add(executedFiles);
183
93
  fileIndex++;
184
94
  }
185
95
 
@@ -187,7 +97,8 @@ public class SftpFileOutput
187
97
  public void close()
188
98
  {
189
99
  closeCurrentFile();
190
- manager.close();
100
+ // TODO
101
+ sftpUtils.close();
191
102
  }
192
103
 
193
104
  @Override
@@ -198,7 +109,9 @@ public class SftpFileOutput
198
109
  @Override
199
110
  public TaskReport commit()
200
111
  {
201
- return Exec.newTaskReport();
112
+ TaskReport report = Exec.newTaskReport();
113
+ report.set("file_list", fileList);
114
+ return report;
202
115
  }
203
116
 
204
117
  private void closeCurrentFile()
@@ -213,100 +126,8 @@ public class SftpFileOutput
213
126
  }
214
127
  }
215
128
 
216
- private Void uploadFile(final String remotePath)
217
- {
218
- try {
219
- return retryExecutor()
220
- .withRetryLimit(maxConnectionRetry)
221
- .withInitialRetryWait(500)
222
- .withMaxRetryWait(30 * 1000)
223
- .runInterruptible(new Retryable<Void>() {
224
- @Override
225
- public Void call() throws IOException
226
- {
227
- FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
228
- logger.info("new sftp file: {}", remoteFile.getPublicURIString());
229
- try (BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream())) {
230
- try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
231
- IOUtils.copy(inputStream, outputStream);
232
- }
233
- }
234
- return null;
235
- }
236
-
237
- @Override
238
- public boolean isRetryableException(Exception exception)
239
- {
240
- return true;
241
- }
242
-
243
- @Override
244
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
245
- {
246
- String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
247
- retryCount, retryLimit, retryWait / 1000, exception.getMessage());
248
- if (retryCount % 3 == 0) {
249
- logger.warn(message, exception);
250
- }
251
- else {
252
- logger.warn(message);
253
- }
254
- }
255
-
256
- @Override
257
- public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
258
- {
259
- }
260
- });
261
- }
262
- catch (RetryGiveupException ex) {
263
- throw Throwables.propagate(ex.getCause());
264
- }
265
- catch (InterruptedException ex) {
266
- throw Throwables.propagate(ex);
267
- }
268
- }
269
-
270
- private URI getSftpFileUri(String remoteFilePath)
271
- {
272
- try {
273
- return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
274
- }
275
- catch (URISyntaxException e) {
276
- logger.error(e.getMessage());
277
- throw new ConfigException(e);
278
- }
279
- }
280
-
281
129
  private String getOutputFilePath()
282
130
  {
283
131
  return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
284
132
  }
285
-
286
- private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
287
- {
288
- FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
289
- if (file.exists()) {
290
- file.delete();
291
- }
292
- if (file.getParent().exists()) {
293
- logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
294
- }
295
- else {
296
- logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
297
- file.getParent().createFolder();
298
- }
299
- return file;
300
- }
301
-
302
- private Function<LocalFile, String> localFileToPathString()
303
- {
304
- return new Function<LocalFile, String>()
305
- {
306
- public String apply(LocalFile file)
307
- {
308
- return file.getPath().toString();
309
- }
310
- };
311
- }
312
133
  }
@@ -12,15 +12,13 @@ import org.embulk.spi.Exec;
12
12
  import org.embulk.spi.FileOutputPlugin;
13
13
  import org.embulk.spi.TransactionalFileOutput;
14
14
  import org.embulk.spi.unit.LocalFile;
15
- import org.slf4j.Logger;
16
15
 
17
16
  import java.util.List;
17
+ import java.util.Map;
18
18
 
19
19
  public class SftpFileOutputPlugin
20
20
  implements FileOutputPlugin
21
21
  {
22
- private Logger logger = Exec.getLogger(SftpFileOutputPlugin.class);
23
-
24
22
  public interface PluginTask
25
23
  extends Task
26
24
  {
@@ -101,6 +99,17 @@ public class SftpFileOutputPlugin
101
99
  int taskCount,
102
100
  List<TaskReport> successTaskReports)
103
101
  {
102
+ SftpUtils sftpUtils = new SftpUtils(taskSource.loadTask(PluginTask.class));
103
+ for (TaskReport report : successTaskReports) {
104
+ List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
105
+ for (Map<String, String> pairFiles : moveFileList) {
106
+ String temporaryFileName = pairFiles.get("temporary_filename");
107
+ String realFileName = pairFiles.get("real_filename");
108
+
109
+ sftpUtils.renameFile(temporaryFileName, realFileName);
110
+ }
111
+ }
112
+ sftpUtils.close();
104
113
  }
105
114
 
106
115
  @Override
@@ -0,0 +1,278 @@
1
+ package org.embulk.output.sftp;
2
+
3
+ import com.google.common.base.Function;
4
+ import com.google.common.base.Throwables;
5
+ import org.apache.commons.io.IOUtils;
6
+ import org.apache.commons.vfs2.FileObject;
7
+ import org.apache.commons.vfs2.FileSystemException;
8
+ import org.apache.commons.vfs2.FileSystemOptions;
9
+ import org.apache.commons.vfs2.impl.StandardFileSystemManager;
10
+ import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
11
+ import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
12
+ import org.embulk.config.ConfigException;
13
+ import org.embulk.spi.Exec;
14
+ import org.embulk.spi.unit.LocalFile;
15
+ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
16
+ import org.embulk.spi.util.RetryExecutor.Retryable;
17
+ import org.slf4j.Logger;
18
+
19
+ import java.io.BufferedInputStream;
20
+ import java.io.BufferedOutputStream;
21
+ import java.io.File;
22
+ import java.io.FileInputStream;
23
+ import java.io.IOException;
24
+ import java.net.URI;
25
+ import java.net.URISyntaxException;
26
+
27
+ import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
28
+ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
29
+
30
+ /**
31
+ * Created by takahiro.nakayama on 10/20/15.
32
+ */
33
+ public class SftpUtils
34
+ {
35
+ private final Logger logger = Exec.getLogger(SftpUtils.class);
36
+ private final StandardFileSystemManager manager;
37
+ private final FileSystemOptions fsOptions;
38
+ private final String userInfo;
39
+ private final String host;
40
+ private final int port;
41
+ private final int maxConnectionRetry;
42
+
43
+ private StandardFileSystemManager initializeStandardFileSystemManager()
44
+ {
45
+ if (!logger.isDebugEnabled()) {
46
+ // TODO: change logging format: org.apache.commons.logging.Log
47
+ System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
48
+ }
49
+ StandardFileSystemManager manager = new StandardFileSystemManager();
50
+ manager.setClassLoader(SftpUtils.class.getClassLoader());
51
+ try {
52
+ manager.init();
53
+ }
54
+ catch (FileSystemException e) {
55
+ logger.error(e.getMessage());
56
+ throw new ConfigException(e);
57
+ }
58
+
59
+ return manager;
60
+ }
61
+
62
+ private String initializeUserInfo(PluginTask task)
63
+ {
64
+ String userInfo = task.getUser();
65
+ if (task.getPassword().isPresent()) {
66
+ userInfo += ":" + task.getPassword().get();
67
+ }
68
+ return userInfo;
69
+ }
70
+
71
+ private FileSystemOptions initializeFsOptions(PluginTask task)
72
+ {
73
+ FileSystemOptions fsOptions = new FileSystemOptions();
74
+
75
+ try {
76
+ SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
77
+ builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
78
+ builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
79
+ builder.setStrictHostKeyChecking(fsOptions, "no");
80
+ if (task.getSecretKeyFilePath().isPresent()) {
81
+ IdentityInfo identityInfo = new IdentityInfo(
82
+ new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
83
+ task.getSecretKeyPassphrase().getBytes()
84
+ );
85
+ builder.setIdentityInfo(fsOptions, identityInfo);
86
+ logger.info("set identity: {}", task.getSecretKeyFilePath().get());
87
+ }
88
+
89
+ if (task.getProxy().isPresent()) {
90
+ ProxyTask proxy = task.getProxy().get();
91
+
92
+ ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
93
+
94
+ if (proxy.getHost().isPresent()) {
95
+ builder.setProxyHost(fsOptions, proxy.getHost().get());
96
+ builder.setProxyPort(fsOptions, proxy.getPort());
97
+ }
98
+
99
+ if (proxy.getUser().isPresent()) {
100
+ builder.setProxyUser(fsOptions, proxy.getUser().get());
101
+ }
102
+
103
+ if (proxy.getPassword().isPresent()) {
104
+ builder.setProxyPassword(fsOptions, proxy.getPassword().get());
105
+ }
106
+
107
+ if (proxy.getCommand().isPresent()) {
108
+ builder.setProxyCommand(fsOptions, proxy.getCommand().get());
109
+ }
110
+ }
111
+ }
112
+ catch (FileSystemException e) {
113
+ logger.error(e.getMessage());
114
+ throw new ConfigException(e);
115
+ }
116
+
117
+ return fsOptions;
118
+ }
119
+
120
+ SftpUtils(PluginTask task)
121
+ {
122
+ this.manager = initializeStandardFileSystemManager();
123
+ this.userInfo = initializeUserInfo(task);
124
+ this.fsOptions = initializeFsOptions(task);
125
+ this.host = task.getHost();
126
+ this.port = task.getPort();
127
+ this.maxConnectionRetry = task.getMaxConnectionRetry();
128
+ }
129
+
130
+ public void close()
131
+ {
132
+ manager.close();
133
+ }
134
+
135
+ public Void uploadFile(final File localTempFile, final String remotePath)
136
+ {
137
+ try {
138
+ return retryExecutor()
139
+ .withRetryLimit(maxConnectionRetry)
140
+ .withInitialRetryWait(500)
141
+ .withMaxRetryWait(30 * 1000)
142
+ .runInterruptible(new Retryable<Void>() {
143
+ @Override
144
+ public Void call() throws IOException
145
+ {
146
+ try (FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
147
+ BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
148
+ BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localTempFile))
149
+ ) {
150
+ logger.info("new sftp file: {}", remoteFile.getPublicURIString());
151
+ IOUtils.copy(inputStream, outputStream);
152
+ }
153
+ return null;
154
+ }
155
+
156
+ @Override
157
+ public boolean isRetryableException(Exception exception)
158
+ {
159
+ return true;
160
+ }
161
+
162
+ @Override
163
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
164
+ {
165
+ String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
166
+ retryCount, retryLimit, retryWait / 1000, exception.getMessage());
167
+ if (retryCount % 3 == 0) {
168
+ logger.warn(message, exception);
169
+ }
170
+ else {
171
+ logger.warn(message);
172
+ }
173
+ }
174
+
175
+ @Override
176
+ public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
177
+ {
178
+ }
179
+ });
180
+ }
181
+ catch (RetryGiveupException ex) {
182
+ throw Throwables.propagate(ex.getCause());
183
+ }
184
+ catch (InterruptedException ex) {
185
+ throw Throwables.propagate(ex);
186
+ }
187
+ }
188
+
189
+ public Void renameFile(final String before, final String after)
190
+ {
191
+ try {
192
+ return retryExecutor()
193
+ .withRetryLimit(maxConnectionRetry)
194
+ .withInitialRetryWait(500)
195
+ .withMaxRetryWait(30 * 1000)
196
+ .runInterruptible(new Retryable<Void>() {
197
+ @Override
198
+ public Void call() throws IOException
199
+ {
200
+ FileObject previousFile = manager.resolveFile(getSftpFileUri(before).toString(), fsOptions);
201
+ FileObject afterFile = manager.resolveFile(getSftpFileUri(after).toString(), fsOptions);
202
+ previousFile.moveTo(afterFile);
203
+ logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
204
+
205
+ return null;
206
+ }
207
+
208
+ @Override
209
+ public boolean isRetryableException(Exception exception)
210
+ {
211
+ return true;
212
+ }
213
+
214
+ @Override
215
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
216
+ {
217
+ String message = String.format("SFTP rename remote file failed. Retrying %d/%d after %d seconds. Message: %s",
218
+ retryCount, retryLimit, retryWait / 1000, exception.getMessage());
219
+ if (retryCount % 3 == 0) {
220
+ logger.warn(message, exception);
221
+ }
222
+ else {
223
+ logger.warn(message);
224
+ }
225
+ }
226
+
227
+ @Override
228
+ public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
229
+ {
230
+ }
231
+ });
232
+ }
233
+ catch (RetryGiveupException ex) {
234
+ throw Throwables.propagate(ex.getCause());
235
+ }
236
+ catch (InterruptedException ex) {
237
+ throw Throwables.propagate(ex);
238
+ }
239
+ }
240
+
241
+ private URI getSftpFileUri(String remoteFilePath)
242
+ {
243
+ try {
244
+ return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
245
+ }
246
+ catch (URISyntaxException e) {
247
+ logger.error(e.getMessage());
248
+ throw new ConfigException(e);
249
+ }
250
+ }
251
+
252
+ private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
253
+ {
254
+ FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
255
+ if (file.exists()) {
256
+ file.delete();
257
+ }
258
+ if (file.getParent().exists()) {
259
+ logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
260
+ }
261
+ else {
262
+ logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
263
+ file.getParent().createFolder();
264
+ }
265
+ return file;
266
+ }
267
+
268
+ private Function<LocalFile, String> localFileToPathString()
269
+ {
270
+ return new Function<LocalFile, String>()
271
+ {
272
+ public String apply(LocalFile file)
273
+ {
274
+ return file.getPath().toString();
275
+ }
276
+ };
277
+ }
278
+ }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-sftp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-11 00:00:00.000000000 Z
12
+ date: 2017-11-07 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -66,13 +66,14 @@ files:
66
66
  - src/main/java/org/embulk/output/sftp/ProxyTask.java
67
67
  - src/main/java/org/embulk/output/sftp/SftpFileOutput.java
68
68
  - src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
69
+ - src/main/java/org/embulk/output/sftp/SftpUtils.java
69
70
  - src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
70
71
  - src/test/resources/id_rsa
71
72
  - src/test/resources/id_rsa.pub
72
73
  - classpath/commons-io-2.5.jar
73
74
  - classpath/commons-logging-1.2.jar
74
75
  - classpath/commons-vfs2-2.1.1660580.2.jar
75
- - classpath/embulk-output-sftp-0.1.2.jar
76
+ - classpath/embulk-output-sftp-0.1.3.jar
76
77
  - classpath/jsch-0.1.53.jar
77
78
  homepage: https://github.com/embulk/embulk-output-sftp
78
79
  licenses:
Binary file