embulk-output-sftp 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +6 -1
- data/CHANGELOG.md +4 -0
- data/build.gradle +1 -1
- data/classpath/embulk-output-sftp-0.1.3.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.jar +0 -0
- data/gradle/wrapper/gradle-wrapper.properties +2 -2
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +21 -200
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +12 -3
- data/src/main/java/org/embulk/output/sftp/SftpUtils.java +278 -0
- metadata +4 -3
- data/classpath/embulk-output-sftp-0.1.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c59819e8adf5918b012907c1649584986ca9f77b
|
4
|
+
data.tar.gz: b942eb8bcaf5a55f18feae4ae68a6c0d4ef96b92
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0abe7dba22a634303bc872806d348c2f2f380257c4ebf7cdd5de3e10bf56644e37e82e5ce72e49f8ac2c76169a77781c5aefdf96112f8bb28da37db8178cf97e
|
7
|
+
data.tar.gz: 4e4073715b98d80bf66fd1c24f846079a4faf73993ec7e619355309c2728bac6459183d053c45fcbb80d2cfbccc527ae466324f264d8260b1a52704e227603af
|
data/.travis.yml
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
dist: precise
|
1
2
|
language: java
|
2
3
|
jdk:
|
3
4
|
- openjdk7
|
@@ -6,4 +7,8 @@ jdk:
|
|
6
7
|
script:
|
7
8
|
- ./gradlew test
|
8
9
|
after_success:
|
9
|
-
- ./gradlew jacocoTestReport coveralls
|
10
|
+
- ./gradlew jacocoTestReport coveralls
|
11
|
+
addons:
|
12
|
+
hosts:
|
13
|
+
- example.com
|
14
|
+
hostname: example.com
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
0.1.3 (2017-11-07)
|
2
|
+
- Enhance: Create temporary file and rename it after upload completed
|
3
|
+
- https://github.com/embulk/embulk-output-sftp/pull/37
|
4
|
+
|
1
5
|
0.1.2 (2017-07-11)
|
2
6
|
- Fix: Increment fileIndex after complete uploadFile
|
3
7
|
- https://github.com/embulk/embulk-output-sftp/pull/35
|
data/build.gradle
CHANGED
Binary file
|
Binary file
|
@@ -1,6 +1,6 @@
|
|
1
|
-
#
|
1
|
+
#Sun Jan 08 00:35:58 PST 2017
|
2
2
|
distributionBase=GRADLE_USER_HOME
|
3
3
|
distributionPath=wrapper/dists
|
4
4
|
zipStoreBase=GRADLE_USER_HOME
|
5
5
|
zipStorePath=wrapper/dists
|
6
|
-
distributionUrl=https\://services.gradle.org/distributions/gradle-2.
|
6
|
+
distributionUrl=https\://services.gradle.org/distributions/gradle-3.2.1-bin.zip
|
@@ -1,37 +1,24 @@
|
|
1
1
|
package org.embulk.output.sftp;
|
2
2
|
|
3
|
-
import com.google.common.base.Function;
|
4
3
|
import com.google.common.base.Throwables;
|
5
|
-
import org.apache.commons.io.IOUtils;
|
6
|
-
import org.apache.commons.vfs2.FileObject;
|
7
|
-
import org.apache.commons.vfs2.FileSystemException;
|
8
|
-
import org.apache.commons.vfs2.FileSystemOptions;
|
9
|
-
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
10
|
-
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
11
|
-
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
12
|
-
import org.embulk.config.ConfigException;
|
13
4
|
import org.embulk.config.TaskReport;
|
14
5
|
import org.embulk.spi.Buffer;
|
15
6
|
import org.embulk.spi.Exec;
|
16
7
|
import org.embulk.spi.FileOutput;
|
17
8
|
import org.embulk.spi.TransactionalFileOutput;
|
18
|
-
import org.embulk.spi.unit.LocalFile;
|
19
|
-
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
20
|
-
import org.embulk.spi.util.RetryExecutor.Retryable;
|
21
9
|
import org.slf4j.Logger;
|
22
10
|
|
23
|
-
import java.io.BufferedInputStream;
|
24
11
|
import java.io.BufferedOutputStream;
|
25
12
|
import java.io.File;
|
26
|
-
import java.io.FileInputStream;
|
27
13
|
import java.io.FileNotFoundException;
|
28
14
|
import java.io.FileOutputStream;
|
29
15
|
import java.io.IOException;
|
30
|
-
import java.
|
31
|
-
import java.
|
16
|
+
import java.util.ArrayList;
|
17
|
+
import java.util.HashMap;
|
18
|
+
import java.util.List;
|
19
|
+
import java.util.Map;
|
32
20
|
|
33
21
|
import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
|
34
|
-
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
35
22
|
|
36
23
|
/**
|
37
24
|
* Created by takahiro.nakayama on 10/20/15.
|
@@ -40,110 +27,26 @@ public class SftpFileOutput
|
|
40
27
|
implements FileOutput, TransactionalFileOutput
|
41
28
|
{
|
42
29
|
private final Logger logger = Exec.getLogger(SftpFileOutput.class);
|
43
|
-
private final StandardFileSystemManager manager;
|
44
|
-
private final FileSystemOptions fsOptions;
|
45
|
-
private final String userInfo;
|
46
|
-
private final String host;
|
47
|
-
private final int port;
|
48
|
-
private final int maxConnectionRetry;
|
49
30
|
private final String pathPrefix;
|
50
31
|
private final String sequenceFormat;
|
51
32
|
private final String fileNameExtension;
|
52
33
|
|
53
34
|
private final int taskIndex;
|
35
|
+
private final SftpUtils sftpUtils;
|
54
36
|
private int fileIndex = 0;
|
55
37
|
private File tempFile;
|
56
38
|
private BufferedOutputStream localOutput = null;
|
39
|
+
private List<Map<String, String>> fileList = new ArrayList<>();
|
57
40
|
|
58
|
-
private
|
59
|
-
{
|
60
|
-
if (!logger.isDebugEnabled()) {
|
61
|
-
// TODO: change logging format: org.apache.commons.logging.Log
|
62
|
-
System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
|
63
|
-
}
|
64
|
-
StandardFileSystemManager manager = new StandardFileSystemManager();
|
65
|
-
manager.setClassLoader(SftpFileOutput.class.getClassLoader());
|
66
|
-
try {
|
67
|
-
manager.init();
|
68
|
-
}
|
69
|
-
catch (FileSystemException e) {
|
70
|
-
logger.error(e.getMessage());
|
71
|
-
throw new ConfigException(e);
|
72
|
-
}
|
73
|
-
|
74
|
-
return manager;
|
75
|
-
}
|
76
|
-
|
77
|
-
private String initializeUserInfo(PluginTask task)
|
78
|
-
{
|
79
|
-
String userInfo = task.getUser();
|
80
|
-
if (task.getPassword().isPresent()) {
|
81
|
-
userInfo += ":" + task.getPassword().get();
|
82
|
-
}
|
83
|
-
return userInfo;
|
84
|
-
}
|
85
|
-
|
86
|
-
private FileSystemOptions initializeFsOptions(PluginTask task)
|
87
|
-
{
|
88
|
-
FileSystemOptions fsOptions = new FileSystemOptions();
|
89
|
-
|
90
|
-
try {
|
91
|
-
SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
|
92
|
-
builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
|
93
|
-
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
|
94
|
-
builder.setStrictHostKeyChecking(fsOptions, "no");
|
95
|
-
if (task.getSecretKeyFilePath().isPresent()) {
|
96
|
-
IdentityInfo identityInfo = new IdentityInfo(
|
97
|
-
new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
|
98
|
-
task.getSecretKeyPassphrase().getBytes()
|
99
|
-
);
|
100
|
-
builder.setIdentityInfo(fsOptions, identityInfo);
|
101
|
-
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
102
|
-
}
|
103
|
-
|
104
|
-
if (task.getProxy().isPresent()) {
|
105
|
-
ProxyTask proxy = task.getProxy().get();
|
106
|
-
|
107
|
-
ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
|
108
|
-
|
109
|
-
if (proxy.getHost().isPresent()) {
|
110
|
-
builder.setProxyHost(fsOptions, proxy.getHost().get());
|
111
|
-
builder.setProxyPort(fsOptions, proxy.getPort());
|
112
|
-
}
|
113
|
-
|
114
|
-
if (proxy.getUser().isPresent()) {
|
115
|
-
builder.setProxyUser(fsOptions, proxy.getUser().get());
|
116
|
-
}
|
117
|
-
|
118
|
-
if (proxy.getPassword().isPresent()) {
|
119
|
-
builder.setProxyPassword(fsOptions, proxy.getPassword().get());
|
120
|
-
}
|
121
|
-
|
122
|
-
if (proxy.getCommand().isPresent()) {
|
123
|
-
builder.setProxyCommand(fsOptions, proxy.getCommand().get());
|
124
|
-
}
|
125
|
-
}
|
126
|
-
}
|
127
|
-
catch (FileSystemException e) {
|
128
|
-
logger.error(e.getMessage());
|
129
|
-
throw new ConfigException(e);
|
130
|
-
}
|
131
|
-
|
132
|
-
return fsOptions;
|
133
|
-
}
|
41
|
+
private final String temporaryFileSuffix = ".tmp";
|
134
42
|
|
135
43
|
SftpFileOutput(PluginTask task, int taskIndex)
|
136
44
|
{
|
137
|
-
this.manager = initializeStandardFileSystemManager();
|
138
|
-
this.userInfo = initializeUserInfo(task);
|
139
|
-
this.fsOptions = initializeFsOptions(task);
|
140
|
-
this.host = task.getHost();
|
141
|
-
this.port = task.getPort();
|
142
|
-
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
143
45
|
this.pathPrefix = task.getPathPrefix();
|
144
46
|
this.sequenceFormat = task.getSequenceFormat();
|
145
47
|
this.fileNameExtension = task.getFileNameExtension();
|
146
48
|
this.taskIndex = taskIndex;
|
49
|
+
this.sftpUtils = new SftpUtils(task);
|
147
50
|
}
|
148
51
|
|
149
52
|
@Override
|
@@ -179,7 +82,14 @@ public class SftpFileOutput
|
|
179
82
|
public void finish()
|
180
83
|
{
|
181
84
|
closeCurrentFile();
|
182
|
-
|
85
|
+
String fileName = getOutputFilePath();
|
86
|
+
String temporaryFileName = fileName + temporaryFileSuffix;
|
87
|
+
sftpUtils.uploadFile(tempFile, temporaryFileName);
|
88
|
+
|
89
|
+
Map<String, String> executedFiles = new HashMap<>();
|
90
|
+
executedFiles.put("temporary_filename", temporaryFileName);
|
91
|
+
executedFiles.put("real_filename", fileName);
|
92
|
+
fileList.add(executedFiles);
|
183
93
|
fileIndex++;
|
184
94
|
}
|
185
95
|
|
@@ -187,7 +97,8 @@ public class SftpFileOutput
|
|
187
97
|
public void close()
|
188
98
|
{
|
189
99
|
closeCurrentFile();
|
190
|
-
|
100
|
+
// TODO
|
101
|
+
sftpUtils.close();
|
191
102
|
}
|
192
103
|
|
193
104
|
@Override
|
@@ -198,7 +109,9 @@ public class SftpFileOutput
|
|
198
109
|
@Override
|
199
110
|
public TaskReport commit()
|
200
111
|
{
|
201
|
-
|
112
|
+
TaskReport report = Exec.newTaskReport();
|
113
|
+
report.set("file_list", fileList);
|
114
|
+
return report;
|
202
115
|
}
|
203
116
|
|
204
117
|
private void closeCurrentFile()
|
@@ -213,100 +126,8 @@ public class SftpFileOutput
|
|
213
126
|
}
|
214
127
|
}
|
215
128
|
|
216
|
-
private Void uploadFile(final String remotePath)
|
217
|
-
{
|
218
|
-
try {
|
219
|
-
return retryExecutor()
|
220
|
-
.withRetryLimit(maxConnectionRetry)
|
221
|
-
.withInitialRetryWait(500)
|
222
|
-
.withMaxRetryWait(30 * 1000)
|
223
|
-
.runInterruptible(new Retryable<Void>() {
|
224
|
-
@Override
|
225
|
-
public Void call() throws IOException
|
226
|
-
{
|
227
|
-
FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
|
228
|
-
logger.info("new sftp file: {}", remoteFile.getPublicURIString());
|
229
|
-
try (BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream())) {
|
230
|
-
try (BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(tempFile))) {
|
231
|
-
IOUtils.copy(inputStream, outputStream);
|
232
|
-
}
|
233
|
-
}
|
234
|
-
return null;
|
235
|
-
}
|
236
|
-
|
237
|
-
@Override
|
238
|
-
public boolean isRetryableException(Exception exception)
|
239
|
-
{
|
240
|
-
return true;
|
241
|
-
}
|
242
|
-
|
243
|
-
@Override
|
244
|
-
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
|
245
|
-
{
|
246
|
-
String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
|
247
|
-
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
248
|
-
if (retryCount % 3 == 0) {
|
249
|
-
logger.warn(message, exception);
|
250
|
-
}
|
251
|
-
else {
|
252
|
-
logger.warn(message);
|
253
|
-
}
|
254
|
-
}
|
255
|
-
|
256
|
-
@Override
|
257
|
-
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
258
|
-
{
|
259
|
-
}
|
260
|
-
});
|
261
|
-
}
|
262
|
-
catch (RetryGiveupException ex) {
|
263
|
-
throw Throwables.propagate(ex.getCause());
|
264
|
-
}
|
265
|
-
catch (InterruptedException ex) {
|
266
|
-
throw Throwables.propagate(ex);
|
267
|
-
}
|
268
|
-
}
|
269
|
-
|
270
|
-
private URI getSftpFileUri(String remoteFilePath)
|
271
|
-
{
|
272
|
-
try {
|
273
|
-
return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
|
274
|
-
}
|
275
|
-
catch (URISyntaxException e) {
|
276
|
-
logger.error(e.getMessage());
|
277
|
-
throw new ConfigException(e);
|
278
|
-
}
|
279
|
-
}
|
280
|
-
|
281
129
|
private String getOutputFilePath()
|
282
130
|
{
|
283
131
|
return pathPrefix + String.format(sequenceFormat, taskIndex, fileIndex) + fileNameExtension;
|
284
132
|
}
|
285
|
-
|
286
|
-
private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
|
287
|
-
{
|
288
|
-
FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
|
289
|
-
if (file.exists()) {
|
290
|
-
file.delete();
|
291
|
-
}
|
292
|
-
if (file.getParent().exists()) {
|
293
|
-
logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
|
294
|
-
}
|
295
|
-
else {
|
296
|
-
logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
|
297
|
-
file.getParent().createFolder();
|
298
|
-
}
|
299
|
-
return file;
|
300
|
-
}
|
301
|
-
|
302
|
-
private Function<LocalFile, String> localFileToPathString()
|
303
|
-
{
|
304
|
-
return new Function<LocalFile, String>()
|
305
|
-
{
|
306
|
-
public String apply(LocalFile file)
|
307
|
-
{
|
308
|
-
return file.getPath().toString();
|
309
|
-
}
|
310
|
-
};
|
311
|
-
}
|
312
133
|
}
|
@@ -12,15 +12,13 @@ import org.embulk.spi.Exec;
|
|
12
12
|
import org.embulk.spi.FileOutputPlugin;
|
13
13
|
import org.embulk.spi.TransactionalFileOutput;
|
14
14
|
import org.embulk.spi.unit.LocalFile;
|
15
|
-
import org.slf4j.Logger;
|
16
15
|
|
17
16
|
import java.util.List;
|
17
|
+
import java.util.Map;
|
18
18
|
|
19
19
|
public class SftpFileOutputPlugin
|
20
20
|
implements FileOutputPlugin
|
21
21
|
{
|
22
|
-
private Logger logger = Exec.getLogger(SftpFileOutputPlugin.class);
|
23
|
-
|
24
22
|
public interface PluginTask
|
25
23
|
extends Task
|
26
24
|
{
|
@@ -101,6 +99,17 @@ public class SftpFileOutputPlugin
|
|
101
99
|
int taskCount,
|
102
100
|
List<TaskReport> successTaskReports)
|
103
101
|
{
|
102
|
+
SftpUtils sftpUtils = new SftpUtils(taskSource.loadTask(PluginTask.class));
|
103
|
+
for (TaskReport report : successTaskReports) {
|
104
|
+
List<Map<String, String>> moveFileList = report.get(List.class, "file_list");
|
105
|
+
for (Map<String, String> pairFiles : moveFileList) {
|
106
|
+
String temporaryFileName = pairFiles.get("temporary_filename");
|
107
|
+
String realFileName = pairFiles.get("real_filename");
|
108
|
+
|
109
|
+
sftpUtils.renameFile(temporaryFileName, realFileName);
|
110
|
+
}
|
111
|
+
}
|
112
|
+
sftpUtils.close();
|
104
113
|
}
|
105
114
|
|
106
115
|
@Override
|
@@ -0,0 +1,278 @@
|
|
1
|
+
package org.embulk.output.sftp;
|
2
|
+
|
3
|
+
import com.google.common.base.Function;
|
4
|
+
import com.google.common.base.Throwables;
|
5
|
+
import org.apache.commons.io.IOUtils;
|
6
|
+
import org.apache.commons.vfs2.FileObject;
|
7
|
+
import org.apache.commons.vfs2.FileSystemException;
|
8
|
+
import org.apache.commons.vfs2.FileSystemOptions;
|
9
|
+
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
10
|
+
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
11
|
+
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
12
|
+
import org.embulk.config.ConfigException;
|
13
|
+
import org.embulk.spi.Exec;
|
14
|
+
import org.embulk.spi.unit.LocalFile;
|
15
|
+
import org.embulk.spi.util.RetryExecutor.RetryGiveupException;
|
16
|
+
import org.embulk.spi.util.RetryExecutor.Retryable;
|
17
|
+
import org.slf4j.Logger;
|
18
|
+
|
19
|
+
import java.io.BufferedInputStream;
|
20
|
+
import java.io.BufferedOutputStream;
|
21
|
+
import java.io.File;
|
22
|
+
import java.io.FileInputStream;
|
23
|
+
import java.io.IOException;
|
24
|
+
import java.net.URI;
|
25
|
+
import java.net.URISyntaxException;
|
26
|
+
|
27
|
+
import static org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
|
28
|
+
import static org.embulk.spi.util.RetryExecutor.retryExecutor;
|
29
|
+
|
30
|
+
/**
|
31
|
+
* Created by takahiro.nakayama on 10/20/15.
|
32
|
+
*/
|
33
|
+
public class SftpUtils
|
34
|
+
{
|
35
|
+
private final Logger logger = Exec.getLogger(SftpUtils.class);
|
36
|
+
private final StandardFileSystemManager manager;
|
37
|
+
private final FileSystemOptions fsOptions;
|
38
|
+
private final String userInfo;
|
39
|
+
private final String host;
|
40
|
+
private final int port;
|
41
|
+
private final int maxConnectionRetry;
|
42
|
+
|
43
|
+
private StandardFileSystemManager initializeStandardFileSystemManager()
|
44
|
+
{
|
45
|
+
if (!logger.isDebugEnabled()) {
|
46
|
+
// TODO: change logging format: org.apache.commons.logging.Log
|
47
|
+
System.setProperty("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");
|
48
|
+
}
|
49
|
+
StandardFileSystemManager manager = new StandardFileSystemManager();
|
50
|
+
manager.setClassLoader(SftpUtils.class.getClassLoader());
|
51
|
+
try {
|
52
|
+
manager.init();
|
53
|
+
}
|
54
|
+
catch (FileSystemException e) {
|
55
|
+
logger.error(e.getMessage());
|
56
|
+
throw new ConfigException(e);
|
57
|
+
}
|
58
|
+
|
59
|
+
return manager;
|
60
|
+
}
|
61
|
+
|
62
|
+
private String initializeUserInfo(PluginTask task)
|
63
|
+
{
|
64
|
+
String userInfo = task.getUser();
|
65
|
+
if (task.getPassword().isPresent()) {
|
66
|
+
userInfo += ":" + task.getPassword().get();
|
67
|
+
}
|
68
|
+
return userInfo;
|
69
|
+
}
|
70
|
+
|
71
|
+
private FileSystemOptions initializeFsOptions(PluginTask task)
|
72
|
+
{
|
73
|
+
FileSystemOptions fsOptions = new FileSystemOptions();
|
74
|
+
|
75
|
+
try {
|
76
|
+
SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
|
77
|
+
builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
|
78
|
+
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
|
79
|
+
builder.setStrictHostKeyChecking(fsOptions, "no");
|
80
|
+
if (task.getSecretKeyFilePath().isPresent()) {
|
81
|
+
IdentityInfo identityInfo = new IdentityInfo(
|
82
|
+
new File((task.getSecretKeyFilePath().transform(localFileToPathString()).get())),
|
83
|
+
task.getSecretKeyPassphrase().getBytes()
|
84
|
+
);
|
85
|
+
builder.setIdentityInfo(fsOptions, identityInfo);
|
86
|
+
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
87
|
+
}
|
88
|
+
|
89
|
+
if (task.getProxy().isPresent()) {
|
90
|
+
ProxyTask proxy = task.getProxy().get();
|
91
|
+
|
92
|
+
ProxyTask.ProxyType.setProxyType(builder, fsOptions, proxy.getType());
|
93
|
+
|
94
|
+
if (proxy.getHost().isPresent()) {
|
95
|
+
builder.setProxyHost(fsOptions, proxy.getHost().get());
|
96
|
+
builder.setProxyPort(fsOptions, proxy.getPort());
|
97
|
+
}
|
98
|
+
|
99
|
+
if (proxy.getUser().isPresent()) {
|
100
|
+
builder.setProxyUser(fsOptions, proxy.getUser().get());
|
101
|
+
}
|
102
|
+
|
103
|
+
if (proxy.getPassword().isPresent()) {
|
104
|
+
builder.setProxyPassword(fsOptions, proxy.getPassword().get());
|
105
|
+
}
|
106
|
+
|
107
|
+
if (proxy.getCommand().isPresent()) {
|
108
|
+
builder.setProxyCommand(fsOptions, proxy.getCommand().get());
|
109
|
+
}
|
110
|
+
}
|
111
|
+
}
|
112
|
+
catch (FileSystemException e) {
|
113
|
+
logger.error(e.getMessage());
|
114
|
+
throw new ConfigException(e);
|
115
|
+
}
|
116
|
+
|
117
|
+
return fsOptions;
|
118
|
+
}
|
119
|
+
|
120
|
+
SftpUtils(PluginTask task)
|
121
|
+
{
|
122
|
+
this.manager = initializeStandardFileSystemManager();
|
123
|
+
this.userInfo = initializeUserInfo(task);
|
124
|
+
this.fsOptions = initializeFsOptions(task);
|
125
|
+
this.host = task.getHost();
|
126
|
+
this.port = task.getPort();
|
127
|
+
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
128
|
+
}
|
129
|
+
|
130
|
+
public void close()
|
131
|
+
{
|
132
|
+
manager.close();
|
133
|
+
}
|
134
|
+
|
135
|
+
public Void uploadFile(final File localTempFile, final String remotePath)
|
136
|
+
{
|
137
|
+
try {
|
138
|
+
return retryExecutor()
|
139
|
+
.withRetryLimit(maxConnectionRetry)
|
140
|
+
.withInitialRetryWait(500)
|
141
|
+
.withMaxRetryWait(30 * 1000)
|
142
|
+
.runInterruptible(new Retryable<Void>() {
|
143
|
+
@Override
|
144
|
+
public Void call() throws IOException
|
145
|
+
{
|
146
|
+
try (FileObject remoteFile = newSftpFile(getSftpFileUri(remotePath));
|
147
|
+
BufferedOutputStream outputStream = new BufferedOutputStream(remoteFile.getContent().getOutputStream());
|
148
|
+
BufferedInputStream inputStream = new BufferedInputStream(new FileInputStream(localTempFile))
|
149
|
+
) {
|
150
|
+
logger.info("new sftp file: {}", remoteFile.getPublicURIString());
|
151
|
+
IOUtils.copy(inputStream, outputStream);
|
152
|
+
}
|
153
|
+
return null;
|
154
|
+
}
|
155
|
+
|
156
|
+
@Override
|
157
|
+
public boolean isRetryableException(Exception exception)
|
158
|
+
{
|
159
|
+
return true;
|
160
|
+
}
|
161
|
+
|
162
|
+
@Override
|
163
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
|
164
|
+
{
|
165
|
+
String message = String.format("SFTP output failed. Retrying %d/%d after %d seconds. Message: %s",
|
166
|
+
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
167
|
+
if (retryCount % 3 == 0) {
|
168
|
+
logger.warn(message, exception);
|
169
|
+
}
|
170
|
+
else {
|
171
|
+
logger.warn(message);
|
172
|
+
}
|
173
|
+
}
|
174
|
+
|
175
|
+
@Override
|
176
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
177
|
+
{
|
178
|
+
}
|
179
|
+
});
|
180
|
+
}
|
181
|
+
catch (RetryGiveupException ex) {
|
182
|
+
throw Throwables.propagate(ex.getCause());
|
183
|
+
}
|
184
|
+
catch (InterruptedException ex) {
|
185
|
+
throw Throwables.propagate(ex);
|
186
|
+
}
|
187
|
+
}
|
188
|
+
|
189
|
+
public Void renameFile(final String before, final String after)
|
190
|
+
{
|
191
|
+
try {
|
192
|
+
return retryExecutor()
|
193
|
+
.withRetryLimit(maxConnectionRetry)
|
194
|
+
.withInitialRetryWait(500)
|
195
|
+
.withMaxRetryWait(30 * 1000)
|
196
|
+
.runInterruptible(new Retryable<Void>() {
|
197
|
+
@Override
|
198
|
+
public Void call() throws IOException
|
199
|
+
{
|
200
|
+
FileObject previousFile = manager.resolveFile(getSftpFileUri(before).toString(), fsOptions);
|
201
|
+
FileObject afterFile = manager.resolveFile(getSftpFileUri(after).toString(), fsOptions);
|
202
|
+
previousFile.moveTo(afterFile);
|
203
|
+
logger.info("renamed remote file: {} to {}", previousFile.getPublicURIString(), afterFile.getPublicURIString());
|
204
|
+
|
205
|
+
return null;
|
206
|
+
}
|
207
|
+
|
208
|
+
@Override
|
209
|
+
public boolean isRetryableException(Exception exception)
|
210
|
+
{
|
211
|
+
return true;
|
212
|
+
}
|
213
|
+
|
214
|
+
@Override
|
215
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
|
216
|
+
{
|
217
|
+
String message = String.format("SFTP rename remote file failed. Retrying %d/%d after %d seconds. Message: %s",
|
218
|
+
retryCount, retryLimit, retryWait / 1000, exception.getMessage());
|
219
|
+
if (retryCount % 3 == 0) {
|
220
|
+
logger.warn(message, exception);
|
221
|
+
}
|
222
|
+
else {
|
223
|
+
logger.warn(message);
|
224
|
+
}
|
225
|
+
}
|
226
|
+
|
227
|
+
@Override
|
228
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
229
|
+
{
|
230
|
+
}
|
231
|
+
});
|
232
|
+
}
|
233
|
+
catch (RetryGiveupException ex) {
|
234
|
+
throw Throwables.propagate(ex.getCause());
|
235
|
+
}
|
236
|
+
catch (InterruptedException ex) {
|
237
|
+
throw Throwables.propagate(ex);
|
238
|
+
}
|
239
|
+
}
|
240
|
+
|
241
|
+
private URI getSftpFileUri(String remoteFilePath)
|
242
|
+
{
|
243
|
+
try {
|
244
|
+
return new URI("sftp", userInfo, host, port, remoteFilePath, null, null);
|
245
|
+
}
|
246
|
+
catch (URISyntaxException e) {
|
247
|
+
logger.error(e.getMessage());
|
248
|
+
throw new ConfigException(e);
|
249
|
+
}
|
250
|
+
}
|
251
|
+
|
252
|
+
private FileObject newSftpFile(final URI sftpUri) throws FileSystemException
|
253
|
+
{
|
254
|
+
FileObject file = manager.resolveFile(sftpUri.toString(), fsOptions);
|
255
|
+
if (file.exists()) {
|
256
|
+
file.delete();
|
257
|
+
}
|
258
|
+
if (file.getParent().exists()) {
|
259
|
+
logger.info("parent directory {} exists there", file.getParent().getPublicURIString());
|
260
|
+
}
|
261
|
+
else {
|
262
|
+
logger.info("trying to create parent directory {}", file.getParent().getPublicURIString());
|
263
|
+
file.getParent().createFolder();
|
264
|
+
}
|
265
|
+
return file;
|
266
|
+
}
|
267
|
+
|
268
|
+
private Function<LocalFile, String> localFileToPathString()
|
269
|
+
{
|
270
|
+
return new Function<LocalFile, String>()
|
271
|
+
{
|
272
|
+
public String apply(LocalFile file)
|
273
|
+
{
|
274
|
+
return file.getPath().toString();
|
275
|
+
}
|
276
|
+
};
|
277
|
+
}
|
278
|
+
}
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-sftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-07
|
12
|
+
date: 2017-11-07 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -66,13 +66,14 @@ files:
|
|
66
66
|
- src/main/java/org/embulk/output/sftp/ProxyTask.java
|
67
67
|
- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
|
68
68
|
- src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
|
69
|
+
- src/main/java/org/embulk/output/sftp/SftpUtils.java
|
69
70
|
- src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
|
70
71
|
- src/test/resources/id_rsa
|
71
72
|
- src/test/resources/id_rsa.pub
|
72
73
|
- classpath/commons-io-2.5.jar
|
73
74
|
- classpath/commons-logging-1.2.jar
|
74
75
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
75
|
-
- classpath/embulk-output-sftp-0.1.
|
76
|
+
- classpath/embulk-output-sftp-0.1.3.jar
|
76
77
|
- classpath/jsch-0.1.53.jar
|
77
78
|
homepage: https://github.com/embulk/embulk-output-sftp
|
78
79
|
licenses:
|
Binary file
|