embulk-output-sftp 0.1.2 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0c303670ed86cfbbf0185ef25f38bfaffe9034b9
4
- data.tar.gz: f8fcdabd44af66bc695f7d9134f14c41882430b7
3
+ metadata.gz: c59819e8adf5918b012907c1649584986ca9f77b
4
+ data.tar.gz: b942eb8bcaf5a55f18feae4ae68a6c0d4ef96b92
5
5
  SHA512:
6
- metadata.gz: 26531cc33e7c923bea492d3a7c3df607780668780f6c1897491f3c3eef5215844a8a145ee62ab3119ac237382cfb82783c3654a69fbb799af704f65c8653be0b
7
- data.tar.gz: 3bf1a4df8aa6c2b6920ed47200a708b578803a3984301e4a21c66056225415bc55a30dd99da48d496eda5d49efd54d827d454980909a421c2fd726fbf7d7100f
6
+ metadata.gz: 0abe7dba22a634303bc872806d348c2f2f380257c4ebf7cdd5de3e10bf56644e37e82e5ce72e49f8ac2c76169a77781c5aefdf96112f8bb28da37db8178cf97e
7
+ data.tar.gz: 4e4073715b98d80bf66fd1c24f846079a4faf73993ec7e619355309c2728bac6459183d053c45fcbb80d2cfbccc527ae466324f264d8260b1a52704e227603af
data/.travis.yml CHANGED
@@ -1,3 +1,4 @@
1
+ dist: precise
1
2
  language: java
2
3
  jdk:
3
4
  - openjdk7
@@ -6,4 +7,8 @@ jdk:
6
7
  script:
7
8
  - ./gradlew test
8
9
  after_success:
9
- - ./gradlew jacocoTestReport coveralls
10
+ - ./gradlew jacocoTestReport coveralls
11
+ addons:
12
+ hosts:
13
+ - example.com
14
+ hostname: example.com
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ 0.1.3 (2017-11-07)
2
+ - Enhance: Create temporary file and rename it after upload completed
3
+ - https://github.com/embulk/embulk-output-sftp/pull/37
4
+
1
5
  0.1.2 (2017-07-11)
2
6
  - Fix: Increment fileIndex after complete uploadFile
3
7
  - https://github.com/embulk/embulk-output-sftp/pull/35
data/build.gradle CHANGED
@@ -17,7 +17,7 @@ configurations {
17
17
  }
18
18
 
19
19
  group = "org.embulk.output.sftp"
20
- version = "0.1.2"
20
+ version = "0.1.3"
21
21
  sourceCompatibility = 1.7
22
22
  targetCompatibility = 1.7
23
23
 
Binary file
@@ -1,6 +1,6 @@
1
- #Wed Jan 13 12:41:02 JST 2016
1
+ #Sun Jan 08 00:35:58 PST 2017
2
2
  distributionBase=GRADLE_USER_HOME
3
3
  distributionPath=wrapper/dists
4
4
  zipStoreBase=GRADLE_USER_HOME
5
5
  zipStorePath=wrapper/dists
6
- distributionUrl=https\://services.gradle.org/distributions/gradle-2.10-bin.zip
6
+ distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
@@ -1,37 +1,24 @@
1
1
  package org.embulk.output.sftp;
2
2
 
3
- import com.google.common.base.Function;
4
3
  import com.google.common.base.Throwables;
5
- import org.apache.commons.io.IOUtils;
6
- import org.apache.commons.vfs2.FileObject;
7
- import org.apache.commons.vfs2.FileSystemException;
8
- import org.apache.commons.vfs2.FileSystemOptions;
9
- import org.apache.commons.vfs2.impl.StandardFileSystemManager;
10
- import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
11
- import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
12
- import org.embulk.config.ConfigException;
13
4
  import org.embulk.config.TaskReport;
14
5
  import org.embulk.spi.Buffer;
15
6
  import org.embulk.spi.Exec;
16
7
  import org.embulk.spi.FileOutput;
17
8
  import org.embulk.spi.TransactionalFileOutput;
18
- import org.embulk.spi.unit.LocalFile;
19
- import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
20
- import org.embulk.spi.util.RetryExecutor.Retryable;
21
9
  import org.slf4j.Logger;
22
10
 
23
- import java.io.BufferedInputStream;
24
11
  import java.io.BufferedOutputStream;
25
12
  import java.io.File;
26
- import java.io.FileInputStream;
27
13
  import java.io.FileNotFoundException;
28
14
  import java.io.FileOutputStream;
29
15
  import java.io.IOException;
30
- import java.net.URI;
31
- import java.net.URISyntaxException;
16
+ import java.util.ArrayList;
17
+ import java.util.HashMap;
18
+ import java.util.List;
19
+ import java.util.Map;
32
20
 
33
21
  import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
34
- import static org.embulk.spi.util.RetryExecutor.retryExecutor;
35
22
 
36
23
  /**
37
24
  * Created by takahiro.nakayama on 10/20/15.
@@ -40,110 +27,26 @@ public class SftpFileOutput
40
27
  implements FileOutput, TransactionalFileOutput
41
28
  {
42
29
  private final Logger logger = Exec.getLogger(SftpFileOutput.class);
43
- private final StandardFileSystemManager manager;
44
- private final FileSystemOptions fsOptions;
45
- private final String userInfo;
46
- private final String host;
47
- private final int port;
48
- private final int maxConnectionRetry;
49
30
  private final String pathPrefix;
50
31
  private final String sequenceFormat;
51
32
  private final String fileNameExtension;
52
33
 
53
34
  private final int taskIndex;
35
+ private final SftpUtils sftpUtils;
54
36
  private int fileIndex = 0;
55
37
  private File tempFile;
56
38
  private BufferedOutputStream localOutput = null;
39
+ private List<Map<String, String>> fileList = new ArrayList<>();
57
40
 
58
- private StandardFileSystemManager initializeStandardFileSystemManager()
59
- {
60
- if (!logger.isDebugEnabled()) {
61
- // TODO: change logging format: org.apache.commons.logging.Log
62
- System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
63
- }
64
- StandardFileSystemManager manager = new StandardFileSystemManager();
65
- manager.setClassLoader(SftpFileOutput.class.getClassLoader());
66
- try {
67
- manager.init();
68
- }
69
- catch (FileSystemException e) {
70
- logger.error(e.getMessage());
71
- throw new ConfigException(e);
72
- }
73
-
74
- return manager;
75
- }
76
-
77
- private String initializeUserInfo(PluginTask task)
78
- {
79
- String userInfo = task.getUser();
80
- if (task.getPassword().isPresent()) {
81
- userInfo += ":" + task.getPassword().get();
82
- }
83
- return userInfo;
84
- }
85
-
86
- private FileSystemOptions initializeFsOptions(PluginTask task)
87
- {
88
- FileSystemOptions fsOptions = new FileSystemOptions();
89
-
90
- try {
91
- SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
92
- builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
93
- builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
94
- builder.setStrictHostKeyChecking(fsOptions, "no");
95
- if (task.getSecretKeyFilePath().isPresent()) {
96
- IdentityInfo identityInfo = new IdentityInfo(
97
- new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
98
- task.getSecretKeyPassphrase().getBytes()
99
- );
100
- builder.setIdentityInfo(fsOptions, identityInfo);
101
- logger.info("set identity: {}", task.getSecretKeyFilePath().get());
102
- }
103
-
104
- if (task.getProxy().isPresent()) {
105
- ProxyTask proxy = task.getProxy().get();
106
-
107
- ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
108
-
109
- if (proxy.getHost().isPresent()) {
110
- builder.setProxyHost(fsOptions, proxy.getHost().get());
111
- builder.setProxyPort(fsOptions, proxy.getPort());
112
- }
113
-
114
- if (proxy.getUser().isPresent()) {
115
- builder.setProxyUser(fsOptions, proxy.getUser().get());
116
- }
117
-
118
- if (proxy.getPassword().isPresent()) {
119
- builder.setProxyPassword(fsOptions, proxy.getPassword().get());
120
- }
121
-
122
- if (proxy.getCommand().isPresent()) {
123
- builder.setProxyCommand(fsOptions, proxy.getCommand().get());
124
- }
125
- }
126
- }
127
- catch (FileSystemException e) {
128
- logger.error(e.getMessage());
129
- throw new ConfigException(e);
130
- }
131
-
132
- return fsOptions;
133
- }
41
+ private final String temporaryFileSuffix = ".tmp";
134
42
 
135
43
  SftpFileOutput(PluginTask task, int taskIndex)
136
44
  {
137
- this.manager = initializeStandardFileSystemManager();
138
- this.userInfo = initializeUserInfo(task);
139
- this.fsOptions = initializeFsOptions(task);
140
- this.host = task.getHost();
141
- this.port = task.getPort();
142
- this.maxConnectionRetry = task.getMaxConnectionRetry();
143
45
  this.pathPrefix = task.getPathPrefix();
144
46
  this.sequenceFormat = task.getSequenceFormat();
145
47
  this.fileNameExtension = task.getFileNameExtension();
146
48
  this.taskIndex = taskIndex;
49
+ this.sftpUtils = new SftpUtils(task);
147
50
  }
148
51
 
149
52
  @Override
@@ -179,7 +82,14 @@ public class SftpFileOutput
179
82
  public void finish()
180
83
  {
181
84
  closeCurrentFile();
182
- uploadFile(getOutputFilePath());
85
+ String fileName = getOutputFilePath();
86
+ String temporaryFileName = fileName + temporaryFileSuffix;
87
+ sftpUtils.uploadFile(tempFile, temporaryFileName);
88
+
89
+ Map<String, String> executedFiles = new HashMap<>();
90
+ executedFiles.put("temporary_filename", temporaryFileName);
91
+ executedFiles.put("real_filename", fileName);
92
+ fileList.add(executedFiles);
183
93
  fileIndex++;
184
94
  }
185
95
 
@@ -187,7 +97,8 @@ public class SftpFileOutput
187
97
  public void close()
188
98
  {
189
99
  closeCurrentFile();
190
- manager.close();
100
+ // TODO
101
+ sftpUtils.close();
191
102
  }
192
103
 
193
104
  @Override
@@ -198,7 +109,9 @@ public class SftpFileOutput
198
109
  @Override
199
110
  public TaskReport commit()
200
111
  {
201
- return Exec.newTaskReport();
112
+ TaskReport report = Exec.newTaskReport();
113
+ report.set("file_list", fileList);
114
+ return report;
202
115
  }
203
116
 
204
117
  private void closeCurrentFile()
@@ -213,100 +126,8 @@ public class SftpFileOutput
213
126
  }
214
127
  }
215
128
 
216
- private Void uploadFile(final String remotePath)
217
- {
218
- try {
219
- return retryExecutor()
220
- .withRetryLimit(maxConnectionRetry)
221
- .withInitialRetryWait(500)
222
- .withMaxRetryWait(30 * 1000)
223
- .runInterruptible(new Retryable<Void>() {
224
- @Override
225
- public Void call() throws IOException
226
- {
227
- FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
228
- logger.info("new sftp file: {}", remoteFile.getPublicURIString());
229
- try (BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream())) {
230
- try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
231
- IOUtils.copy(inputStream, outputStream);
232
- }
233
- }
234
- return null;
235
- }
236
-
237
- @Override
238
- public boolean isRetryableException(Exception exception)
239
- {
240
- return true;
241
- }
242
-
243
- @Override
244
- public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
245
- {
246
- String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
247
- retryCount, retryLimit, retryWait / 1000, exception.getMessage());
248
- if (retryCount % 3 == 0) {
249
- logger.warn(message, exception);
250
- }
251
- else {
252
- logger.warn(message);
253
- }
254
- }
255
-
256
- @Override
257
- public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
258
- {
259
- }
260
- });
261
- }
262
- catch (RetryGiveupException ex) {
263
- throw Throwables.propagate(ex.getCause());
264
- }
265
- catch (InterruptedException ex) {
266
- throw Throwables.propagate(ex);
267
- }
268
- }
269
-
270
- private URI getSftpFileUri(String remoteFilePath)
271
- {
272
- try {
273
- return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
274
- }
275
- catch (URISyntaxException e) {
276
- logger.error(e.getMessage());
277
- throw new ConfigException(e);
278
- }
279
- }
280
-
281
129
  private String getOutputFilePath()
282
130
  {
283
131
  return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
284
132
  }
285
-
286
- private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
287
- {
288
- FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
289
- if (file.exists()) {
290
- file.delete();
291
- }
292
- if (file.getParent().exists()) {
293
- logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
294
- }
295
- else {
296
- logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
297
- file.getParent().createFolder();
298
- }
299
- return file;
300
- }
301
-
302
- private Function<LocalFile, String> localFileToPathString()
303
- {
304
- return new Function<LocalFile, String>()
305
- {
306
- public String apply(LocalFile file)
307
- {
308
- return file.getPath().toString();
309
- }
310
- };
311
- }
312
133
  }
@@ -12,15 +12,13 @@ import org.embulk.spi.Exec;
12
12
  import org.embulk.spi.FileOutputPlugin;
13
13
  import org.embulk.spi.TransactionalFileOutput;
14
14
  import org.embulk.spi.unit.LocalFile;
15
- import org.slf4j.Logger;
16
15
 
17
16
  import java.util.List;
17
+ import java.util.Map;
18
18
 
19
19
  public class SftpFileOutputPlugin
20
20
  implements FileOutputPlugin
21
21
  {
22
- private Logger logger = Exec.getLogger(SftpFileOutputPlugin.class);
23
-
24
22
  public interface PluginTask
25
23
  extends Task
26
24
  {
@@ -101,6 +99,17 @@ public class SftpFileOutputPlugin
101
99
  int taskCount,
102
100
  List<TaskReport> successTaskReports)
103
101
  {
102
+ SftpUtils sftpUtils = new SftpUtils(taskSource.loadTask(PluginTask.class));
103
+ for (TaskReport report : successTaskReports) {
104
+ List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
105
+ for (Map<String, String> pairFiles : moveFileList) {
106
+ String temporaryFileName = pairFiles.get("temporary_filename");
107
+ String realFileName = pairFiles.get("real_filename");
108
+
109
+ sftpUtils.renameFile(temporaryFileName, realFileName);
110
+ }
111
+ }
112
+ sftpUtils.close();
104
113
  }
105
114
 
106
115
  @Override
@@ -0,0 +1,278 @@
1
+ package org.embulk.output.sftp;
2
+
3
+ import com.google.common.base.Function;
4
+ import com.google.common.base.Throwables;
5
+ import org.apache.commons.io.IOUtils;
6
+ import org.apache.commons.vfs2.FileObject;
7
+ import org.apache.commons.vfs2.FileSystemException;
8
+ import org.apache.commons.vfs2.FileSystemOptions;
9
+ import org.apache.commons.vfs2.impl.StandardFileSystemManager;
10
+ import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
11
+ import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
12
+ import org.embulk.config.ConfigException;
13
+ import org.embulk.spi.Exec;
14
+ import org.embulk.spi.unit.LocalFile;
15
+ import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
16
+ import org.embulk.spi.util.RetryExecutor.Retryable;
17
+ import org.slf4j.Logger;
18
+
19
+ import java.io.BufferedInputStream;
20
+ import java.io.BufferedOutputStream;
21
+ import java.io.File;
22
+ import java.io.FileInputStream;
23
+ import java.io.IOException;
24
+ import java.net.URI;
25
+ import java.net.URISyntaxException;
26
+
27
+ import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
28
+ import static org.embulk.spi.util.RetryExecutor.retryExecutor;
29
+
30
+ /**
31
+ * Created by takahiro.nakayama on 10/20/15.
32
+ */
33
+ public class SftpUtils
34
+ {
35
+ private final Logger logger = Exec.getLogger(SftpUtils.class);
36
+ private final StandardFileSystemManager manager;
37
+ private final FileSystemOptions fsOptions;
38
+ private final String userInfo;
39
+ private final String host;
40
+ private final int port;
41
+ private final int maxConnectionRetry;
42
+
43
+ private StandardFileSystemManager initializeStandardFileSystemManager()
44
+ {
45
+ if (!logger.isDebugEnabled()) {
46
+ // TODO: change logging format: org.apache.commons.logging.Log
47
+ System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
48
+ }
49
+ StandardFileSystemManager manager = new StandardFileSystemManager();
50
+ manager.setClassLoader(SftpUtils.class.getClassLoader());
51
+ try {
52
+ manager.init();
53
+ }
54
+ catch (FileSystemException e) {
55
+ logger.error(e.getMessage());
56
+ throw new ConfigException(e);
57
+ }
58
+
59
+ return manager;
60
+ }
61
+
62
+ private String initializeUserInfo(PluginTask task)
63
+ {
64
+ String userInfo = task.getUser();
65
+ if (task.getPassword().isPresent()) {
66
+ userInfo += ":" + task.getPassword().get();
67
+ }
68
+ return userInfo;
69
+ }
70
+
71
+ private FileSystemOptions initializeFsOptions(PluginTask task)
72
+ {
73
+ FileSystemOptions fsOptions = new FileSystemOptions();
74
+
75
+ try {
76
+ SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
77
+ builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
78
+ builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
79
+ builder.setStrictHostKeyChecking(fsOptions, "no");
80
+ if (task.getSecretKeyFilePath().isPresent()) {
81
+ IdentityInfo identityInfo = new IdentityInfo(
82
+ new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
83
+ task.getSecretKeyPassphrase().getBytes()
84
+ );
85
+ builder.setIdentityInfo(fsOptions, identityInfo);
86
+ logger.info("set identity: {}", task.getSecretKeyFilePath().get());
87
+ }
88
+
89
+ if (task.getProxy().isPresent()) {
90
+ ProxyTask proxy = task.getProxy().get();
91
+
92
+ ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
93
+
94
+ if (proxy.getHost().isPresent()) {
95
+ builder.setProxyHost(fsOptions, proxy.getHost().get());
96
+ builder.setProxyPort(fsOptions, proxy.getPort());
97
+ }
98
+
99
+ if (proxy.getUser().isPresent()) {
100
+ builder.setProxyUser(fsOptions, proxy.getUser().get());
101
+ }
102
+
103
+ if (proxy.getPassword().isPresent()) {
104
+ builder.setProxyPassword(fsOptions, proxy.getPassword().get());
105
+ }
106
+
107
+ if (proxy.getCommand().isPresent()) {
108
+ builder.setProxyCommand(fsOptions, proxy.getCommand().get());
109
+ }
110
+ }
111
+ }
112
+ catch (FileSystemException e) {
113
+ logger.error(e.getMessage());
114
+ throw new ConfigException(e);
115
+ }
116
+
117
+ return fsOptions;
118
+ }
119
+
120
+ SftpUtils(PluginTask task)
121
+ {
122
+ this.manager = initializeStandardFileSystemManager();
123
+ this.userInfo = initializeUserInfo(task);
124
+ this.fsOptions = initializeFsOptions(task);
125
+ this.host = task.getHost();
126
+ this.port = task.getPort();
127
+ this.maxConnectionRetry = task.getMaxConnectionRetry();
128
+ }
129
+
130
+ public void close()
131
+ {
132
+ manager.close();
133
+ }
134
+
135
+ public Void uploadFile(final File localTempFile, final String remotePath)
136
+ {
137
+ try {
138
+ return retryExecutor()
139
+ .withRetryLimit(maxConnectionRetry)
140
+ .withInitialRetryWait(500)
141
+ .withMaxRetryWait(30 * 1000)
142
+ .runInterruptible(new Retryable<Void>() {
143
+ @Override
144
+ public Void call() throws IOException
145
+ {
146
+ try (FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
147
+ BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
148
+ BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localTempFile))
149
+ ) {
150
+ logger.info("new sftp file: {}", remoteFile.getPublicURIString());
151
+ IOUtils.copy(inputStream, outputStream);
152
+ }
153
+ return null;
154
+ }
155
+
156
+ @Override
157
+ public boolean isRetryableException(Exception exception)
158
+ {
159
+ return true;
160
+ }
161
+
162
+ @Override
163
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
164
+ {
165
+ String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
166
+ retryCount, retryLimit, retryWait / 1000, exception.getMessage());
167
+ if (retryCount % 3 == 0) {
168
+ logger.warn(message, exception);
169
+ }
170
+ else {
171
+ logger.warn(message);
172
+ }
173
+ }
174
+
175
+ @Override
176
+ public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
177
+ {
178
+ }
179
+ });
180
+ }
181
+ catch (RetryGiveupException ex) {
182
+ throw Throwables.propagate(ex.getCause());
183
+ }
184
+ catch (InterruptedException ex) {
185
+ throw Throwables.propagate(ex);
186
+ }
187
+ }
188
+
189
+ public Void renameFile(final String before, final String after)
190
+ {
191
+ try {
192
+ return retryExecutor()
193
+ .withRetryLimit(maxConnectionRetry)
194
+ .withInitialRetryWait(500)
195
+ .withMaxRetryWait(30 * 1000)
196
+ .runInterruptible(new Retryable<Void>() {
197
+ @Override
198
+ public Void call() throws IOException
199
+ {
200
+ FileObject previousFile = manager.resolveFile(getSftpFileUri(before).toString(), fsOptions);
201
+ FileObject afterFile = manager.resolveFile(getSftpFileUri(after).toString(), fsOptions);
202
+ previousFile.moveTo(afterFile);
203
+ logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
204
+
205
+ return null;
206
+ }
207
+
208
+ @Override
209
+ public boolean isRetryableException(Exception exception)
210
+ {
211
+ return true;
212
+ }
213
+
214
+ @Override
215
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
216
+ {
217
+ String message = String.format("SFTP rename remote file failed. Retrying %d/%d after %d seconds. Message: %s",
218
+ retryCount, retryLimit, retryWait / 1000, exception.getMessage());
219
+ if (retryCount % 3 == 0) {
220
+ logger.warn(message, exception);
221
+ }
222
+ else {
223
+ logger.warn(message);
224
+ }
225
+ }
226
+
227
+ @Override
228
+ public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
229
+ {
230
+ }
231
+ });
232
+ }
233
+ catch (RetryGiveupException ex) {
234
+ throw Throwables.propagate(ex.getCause());
235
+ }
236
+ catch (InterruptedException ex) {
237
+ throw Throwables.propagate(ex);
238
+ }
239
+ }
240
+
241
+ private URI getSftpFileUri(String remoteFilePath)
242
+ {
243
+ try {
244
+ return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
245
+ }
246
+ catch (URISyntaxException e) {
247
+ logger.error(e.getMessage());
248
+ throw new ConfigException(e);
249
+ }
250
+ }
251
+
252
+ private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
253
+ {
254
+ FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
255
+ if (file.exists()) {
256
+ file.delete();
257
+ }
258
+ if (file.getParent().exists()) {
259
+ logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
260
+ }
261
+ else {
262
+ logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
263
+ file.getParent().createFolder();
264
+ }
265
+ return file;
266
+ }
267
+
268
+ private Function<LocalFile, String> localFileToPathString()
269
+ {
270
+ return new Function<LocalFile, String>()
271
+ {
272
+ public String apply(LocalFile file)
273
+ {
274
+ return file.getPath().toString();
275
+ }
276
+ };
277
+ }
278
+ }
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-sftp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.2
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Civitaspo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-07-11 00:00:00.000000000 Z
12
+ date: 2017-11-07 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  requirement: !ruby/object:Gem::Requirement
@@ -66,13 +66,14 @@ files:
66
66
  - src/main/java/org/embulk/output/sftp/ProxyTask.java
67
67
  - src/main/java/org/embulk/output/sftp/SftpFileOutput.java
68
68
  - src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
69
+ - src/main/java/org/embulk/output/sftp/SftpUtils.java
69
70
  - src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
70
71
  - src/test/resources/id_rsa
71
72
  - src/test/resources/id_rsa.pub
72
73
  - classpath/commons-io-2.5.jar
73
74
  - classpath/commons-logging-1.2.jar
74
75
  - classpath/commons-vfs2-2.1.1660580.2.jar
75
- - classpath/embulk-output-sftp-0.1.2.jar
76
+ - classpath/embulk-output-sftp-0.1.3.jar
76
77
  - classpath/jsch-0.1.53.jar
77
78
  homepage: https://github.com/embulk/embulk-output-sftp
78
79
  licenses:
Binary file