embulk-output-sftp 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/README.md +2 -0
- data/build.gradle +15 -3
- data/classpath/embulk-output-sftp-0.0.3.jar +0 -0
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +24 -2
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +4 -0
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +391 -0
- data/src/test/resources/id_rsa +27 -0
- data/src/test/resources/id_rsa.pub +1 -0
- metadata +6 -3
- data/classpath/embulk-output-sftp-0.0.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 429f131e8ee696e9543c77900ba06e550b7bf9db
|
4
|
+
data.tar.gz: e43a2290c0d7e8b18bee5af81656f688d94e5dd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b126f0b957da348b95622b331e83b62238fccc5ed0768aefa8f626eb895335eee4ed5c1bdb5420655a64fd4ba1cdc43a9c902ec28ba70a09f9d56527c8b5581f
|
7
|
+
data.tar.gz: c41dd77673baa74ff9aad663ac28ebe71ffae50cafcf921a3970bbbdc57ec5b5e4f989fb571fdf680b5724b6c0482316908ad5be295ee1db95f7021a2edee828
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# Sftp file output plugin for Embulk
|
2
|
+
[![Build Status](https://travis-ci.org/civitaspo/embulk-output-sftp.svg)](https://travis-ci.org/civitaspo/embulk-output-sftp)
|
3
|
+
[![Coverage Status](https://coveralls.io/repos/civitaspo/embulk-output-sftp/badge.svg?branch=master&service=github)](https://coveralls.io/github/civitaspo/embulk-output-sftp?branch=master)
|
2
4
|
|
3
5
|
Stores files on a SFTP Server
|
4
6
|
|
data/build.gradle
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
plugins {
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "com.github.kt3k.coveralls" version "2.4.0"
|
5
|
+
id "jacoco"
|
4
6
|
id "java"
|
5
7
|
}
|
6
8
|
import com.github.jrubygradle.JRubyExec
|
@@ -12,17 +14,27 @@ configurations {
|
|
12
14
|
provided
|
13
15
|
}
|
14
16
|
|
15
|
-
version = "0.0.
|
17
|
+
version = "0.0.3"
|
16
18
|
sourceCompatibility = 1.7
|
17
19
|
targetCompatibility = 1.7
|
18
20
|
|
19
21
|
dependencies {
|
20
|
-
compile "org.embulk:embulk-core:0.7
|
21
|
-
provided "org.embulk:embulk-core:0.7
|
22
|
+
compile "org.embulk:embulk-core:0.7.+"
|
23
|
+
provided "org.embulk:embulk-core:0.7.+"
|
22
24
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
23
25
|
compile "org.apache.commons:commons-vfs2:2.+"
|
24
26
|
compile "com.jcraft:jsch:0.1.53"
|
25
27
|
testCompile "junit:junit:4.+"
|
28
|
+
testCompile "org.embulk:embulk-core:0.7.+:tests"
|
29
|
+
testCompile "org.embulk:embulk-standards:0.7.+"
|
30
|
+
testCompile "org.apache.sshd:apache-sshd:1.+"
|
31
|
+
}
|
32
|
+
|
33
|
+
jacocoTestReport {
|
34
|
+
reports {
|
35
|
+
xml.enabled = true // coveralls plugin depends on xml format report
|
36
|
+
html.enabled = true
|
37
|
+
}
|
26
38
|
}
|
27
39
|
|
28
40
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
@@ -4,7 +4,6 @@ import com.google.common.base.Throwables;
|
|
4
4
|
import org.apache.commons.vfs2.FileObject;
|
5
5
|
import org.apache.commons.vfs2.FileSystemException;
|
6
6
|
import org.apache.commons.vfs2.FileSystemOptions;
|
7
|
-
import org.apache.commons.vfs2.Selectors;
|
8
7
|
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
9
8
|
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
10
9
|
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
@@ -35,6 +34,7 @@ public class SftpFileOutput
|
|
35
34
|
private final String userInfo;
|
36
35
|
private final String host;
|
37
36
|
private final int port;
|
37
|
+
private final int maxConnectionRetry;
|
38
38
|
private final String pathPrefix;
|
39
39
|
private final String sequenceFormat;
|
40
40
|
private final String fileNameExtension;
|
@@ -82,6 +82,7 @@ public class SftpFileOutput
|
|
82
82
|
if (task.getSecretKeyFilePath().isPresent()) {
|
83
83
|
IdentityInfo identityInfo = new IdentityInfo(new File((task.getSecretKeyFilePath().get())), task.getSecretKeyPassphrase().getBytes());
|
84
84
|
SftpFileSystemConfigBuilder.getInstance().setIdentityInfo(fsOptions, identityInfo);
|
85
|
+
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
85
86
|
}
|
86
87
|
}
|
87
88
|
catch (FileSystemException e) {
|
@@ -99,6 +100,7 @@ public class SftpFileOutput
|
|
99
100
|
this.fsOptions = initializeFsOptions(task);
|
100
101
|
this.host = task.getHost();
|
101
102
|
this.port = task.getPort();
|
103
|
+
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
102
104
|
this.pathPrefix = task.getPathPrefix();
|
103
105
|
this.sequenceFormat = task.getSequenceFormat();
|
104
106
|
this.fileNameExtension = task.getFileNameExtension();
|
@@ -204,6 +206,26 @@ public class SftpFileOutput
|
|
204
206
|
private FileObject newSftpFile(URI sftpUri)
|
205
207
|
throws FileSystemException
|
206
208
|
{
|
207
|
-
|
209
|
+
int count = 0;
|
210
|
+
while (true) {
|
211
|
+
try {
|
212
|
+
return manager.resolveFile(sftpUri.toString(), fsOptions);
|
213
|
+
}
|
214
|
+
catch (FileSystemException e) {
|
215
|
+
if (++count == maxConnectionRetry) {
|
216
|
+
throw e;
|
217
|
+
}
|
218
|
+
logger.warn("failed to connect sftp server: " + e.getMessage(), e);
|
219
|
+
|
220
|
+
try {
|
221
|
+
Thread.sleep(count * 1000); // milliseconds
|
222
|
+
}
|
223
|
+
catch (InterruptedException e1) {
|
224
|
+
// Ignore this exception
|
225
|
+
logger.warn(e.getMessage(), e);
|
226
|
+
}
|
227
|
+
logger.warn("retry to connect sftp server: " + count + " times");
|
228
|
+
}
|
229
|
+
}
|
208
230
|
}
|
209
231
|
}
|
@@ -53,6 +53,10 @@ public class SftpFileOutputPlugin
|
|
53
53
|
@ConfigDefault("600") // 10 minutes
|
54
54
|
public int getSftpConnectionTimeout();
|
55
55
|
|
56
|
+
@Config("max_connection_retry")
|
57
|
+
@ConfigDefault("5") // 5 times retry to connect sftp server if failed.
|
58
|
+
public int getMaxConnectionRetry();
|
59
|
+
|
56
60
|
@Config("path_prefix")
|
57
61
|
public String getPathPrefix();
|
58
62
|
|
@@ -1,5 +1,396 @@
|
|
1
1
|
package org.embulk.output.sftp;
|
2
2
|
|
3
|
+
import com.google.common.base.Charsets;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.google.common.base.Splitter;
|
6
|
+
import com.google.common.base.Throwables;
|
7
|
+
import com.google.common.collect.Lists;
|
8
|
+
import com.google.common.io.Resources;
|
9
|
+
import org.apache.commons.vfs2.FileSystemException;
|
10
|
+
import org.apache.sshd.common.NamedFactory;
|
11
|
+
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
|
12
|
+
import org.apache.sshd.server.Command;
|
13
|
+
import org.apache.sshd.server.SshServer;
|
14
|
+
import org.apache.sshd.server.auth.password.PasswordAuthenticator;
|
15
|
+
import org.apache.sshd.server.auth.pubkey.PublickeyAuthenticator;
|
16
|
+
import org.apache.sshd.server.command.ScpCommandFactory;
|
17
|
+
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
|
18
|
+
import org.apache.sshd.server.session.ServerSession;
|
19
|
+
import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory;
|
20
|
+
import org.embulk.EmbulkTestRuntime;
|
21
|
+
import org.embulk.config.ConfigLoader;
|
22
|
+
import org.embulk.config.ConfigSource;
|
23
|
+
import org.embulk.config.TaskReport;
|
24
|
+
import org.embulk.config.TaskSource;
|
25
|
+
import org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
|
26
|
+
import org.embulk.spi.Exec;
|
27
|
+
import org.embulk.spi.FileOutputRunner;
|
28
|
+
import org.embulk.spi.OutputPlugin.Control;
|
29
|
+
import org.embulk.spi.Page;
|
30
|
+
import org.embulk.spi.PageTestUtils;
|
31
|
+
import org.embulk.spi.Schema;
|
32
|
+
import org.embulk.spi.TransactionalPageOutput;
|
33
|
+
import org.embulk.spi.time.Timestamp;
|
34
|
+
import org.hamcrest.CoreMatchers;
|
35
|
+
import org.hamcrest.Matcher;
|
36
|
+
import org.junit.After;
|
37
|
+
import org.junit.Before;
|
38
|
+
import org.junit.Rule;
|
39
|
+
import org.junit.Test;
|
40
|
+
import org.junit.rules.ExpectedException;
|
41
|
+
import org.junit.rules.TemporaryFolder;
|
42
|
+
import org.slf4j.Logger;
|
43
|
+
|
44
|
+
import java.io.File;
|
45
|
+
import java.io.IOException;
|
46
|
+
import java.nio.file.DirectoryStream;
|
47
|
+
import java.nio.file.FileSystems;
|
48
|
+
import java.nio.file.Files;
|
49
|
+
import java.nio.file.Path;
|
50
|
+
import java.nio.file.PathMatcher;
|
51
|
+
import java.nio.file.Paths;
|
52
|
+
import java.security.PublicKey;
|
53
|
+
import java.util.Arrays;
|
54
|
+
import java.util.Collections;
|
55
|
+
import java.util.List;
|
56
|
+
|
57
|
+
import static com.google.common.io.Files.readLines;
|
58
|
+
import static org.embulk.spi.type.Types.*;
|
59
|
+
import static org.hamcrest.CoreMatchers.containsString;
|
60
|
+
import static org.hamcrest.CoreMatchers.hasItem;
|
61
|
+
import static org.hamcrest.CoreMatchers.instanceOf;
|
62
|
+
import static org.hamcrest.MatcherAssert.*;
|
63
|
+
import static org.hamcrest.core.Is.is;
|
64
|
+
import static org.junit.Assert.assertEquals;
|
65
|
+
//import static org.hamcrest.Matchers.*;
|
66
|
+
|
3
67
|
public class TestSftpFileOutputPlugin
|
4
68
|
{
|
69
|
+
@Rule
|
70
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
71
|
+
|
72
|
+
@Rule
|
73
|
+
public ExpectedException exception = ExpectedException.none();
|
74
|
+
|
75
|
+
@Rule
|
76
|
+
public TemporaryFolder testFolder = new TemporaryFolder();
|
77
|
+
|
78
|
+
private Logger logger = runtime.getExec().getLogger(TestSftpFileOutputPlugin.class);
|
79
|
+
private FileOutputRunner runner;
|
80
|
+
private SshServer sshServer;
|
81
|
+
private static final String HOST = "127.0.0.1";
|
82
|
+
private static final int PORT = 20022;
|
83
|
+
private static final String USERNAME = "username";
|
84
|
+
private static final String PASSWORD = "password";
|
85
|
+
private static final String SECRET_KEY_FILE = Resources.getResource("id_rsa").getPath();
|
86
|
+
private static final String SECRET_KEY_PASSPHRASE = "SECRET_KEY_PASSPHRASE";
|
87
|
+
private static final Schema SCHEMA = new Schema.Builder()
|
88
|
+
.add("_c0", BOOLEAN)
|
89
|
+
.add("_c1", LONG)
|
90
|
+
.add("_c2", DOUBLE)
|
91
|
+
.add("_c3", STRING)
|
92
|
+
.add("_c4", TIMESTAMP)
|
93
|
+
.build();
|
94
|
+
|
95
|
+
@Before
|
96
|
+
public void createResources()
|
97
|
+
throws IOException
|
98
|
+
{
|
99
|
+
// setup the plugin
|
100
|
+
SftpFileOutputPlugin sftpFileOutputPlugin = new SftpFileOutputPlugin();
|
101
|
+
runner = new FileOutputRunner(sftpFileOutputPlugin);
|
102
|
+
|
103
|
+
// setup a mock sftp server
|
104
|
+
sshServer = SshServer.setUpDefaultServer();
|
105
|
+
VirtualFileSystemFactory fsFactory = new VirtualFileSystemFactory();
|
106
|
+
fsFactory.setUserHomeDir(USERNAME, testFolder.getRoot().getAbsolutePath());
|
107
|
+
sshServer.setFileSystemFactory(fsFactory);
|
108
|
+
sshServer.setHost(HOST);
|
109
|
+
sshServer.setPort(PORT);
|
110
|
+
sshServer.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystemFactory()));
|
111
|
+
sshServer.setCommandFactory(new ScpCommandFactory());
|
112
|
+
sshServer.setKeyPairProvider(new SimpleGeneratorHostKeyProvider());
|
113
|
+
sshServer.setPasswordAuthenticator(new PasswordAuthenticator()
|
114
|
+
{
|
115
|
+
@Override
|
116
|
+
public boolean authenticate(final String username, final String password, final ServerSession session)
|
117
|
+
{
|
118
|
+
return USERNAME.contentEquals(username) && PASSWORD.contentEquals(password);
|
119
|
+
}
|
120
|
+
});
|
121
|
+
sshServer.setPublickeyAuthenticator(new PublickeyAuthenticator()
|
122
|
+
{
|
123
|
+
@Override
|
124
|
+
public boolean authenticate(String username, PublicKey key, ServerSession session)
|
125
|
+
{
|
126
|
+
return true;
|
127
|
+
}
|
128
|
+
});
|
129
|
+
|
130
|
+
try {
|
131
|
+
sshServer.start();
|
132
|
+
}
|
133
|
+
catch (IOException e) {
|
134
|
+
logger.debug(e.getMessage(), e);
|
135
|
+
}
|
136
|
+
}
|
137
|
+
|
138
|
+
@After
|
139
|
+
public void cleanup() throws InterruptedException {
|
140
|
+
try {
|
141
|
+
sshServer.stop(true);
|
142
|
+
}
|
143
|
+
catch (Exception e) {
|
144
|
+
logger.debug(e.getMessage(), e);
|
145
|
+
}
|
146
|
+
}
|
147
|
+
|
148
|
+
private ConfigSource getConfigFromYaml(String yaml)
|
149
|
+
{
|
150
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
151
|
+
return loader.fromYamlString(yaml);
|
152
|
+
}
|
153
|
+
|
154
|
+
private List<String> lsR(List<String> fileNames, Path dir) {
|
155
|
+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
|
156
|
+
for (Path path : stream) {
|
157
|
+
if(path.toFile().isDirectory()) {
|
158
|
+
lsR(fileNames, path);
|
159
|
+
} else {
|
160
|
+
fileNames.add(path.toAbsolutePath().toString());
|
161
|
+
}
|
162
|
+
}
|
163
|
+
}
|
164
|
+
catch(IOException e) {
|
165
|
+
logger.debug(e.getMessage(), e);
|
166
|
+
}
|
167
|
+
return fileNames;
|
168
|
+
}
|
169
|
+
|
170
|
+
private void run(String configYaml, final Optional<Integer> sleep)
|
171
|
+
{
|
172
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
173
|
+
runner.transaction(config, SCHEMA, 1, new Control()
|
174
|
+
{
|
175
|
+
@Override
|
176
|
+
public List<TaskReport> run(TaskSource taskSource)
|
177
|
+
{
|
178
|
+
TransactionalPageOutput pageOutput = runner.open(taskSource, SCHEMA, 1);
|
179
|
+
boolean committed = false;
|
180
|
+
try {
|
181
|
+
// Result:
|
182
|
+
// _c0,_c1,_c2,_c3,_c4
|
183
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
184
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
185
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA, true, 2L, 3.0D, "45",
|
186
|
+
Timestamp.ofEpochMilli(678L), true, 2L, 3.0D, "45",
|
187
|
+
Timestamp.ofEpochMilli(678L))) {
|
188
|
+
pageOutput.add(page);
|
189
|
+
if (sleep.isPresent()) {
|
190
|
+
Thread.sleep(sleep.get() * 1000);
|
191
|
+
}
|
192
|
+
}
|
193
|
+
pageOutput.commit();
|
194
|
+
committed = true;
|
195
|
+
}
|
196
|
+
catch (InterruptedException e) {
|
197
|
+
logger.debug(e.getMessage(), e);
|
198
|
+
}
|
199
|
+
finally {
|
200
|
+
if (!committed) {
|
201
|
+
pageOutput.abort();
|
202
|
+
}
|
203
|
+
pageOutput.close();
|
204
|
+
}
|
205
|
+
return Lists.newArrayList();
|
206
|
+
}
|
207
|
+
});
|
208
|
+
}
|
209
|
+
|
210
|
+
private void assertRecordsInFile(String filePath)
|
211
|
+
{
|
212
|
+
try {
|
213
|
+
List<String> lines = readLines(new File(filePath),
|
214
|
+
Charsets.UTF_8);
|
215
|
+
for (int i = 0; i < lines.size(); i++) {
|
216
|
+
String[] record = lines.get(i).split(",");
|
217
|
+
if (i == 0) {
|
218
|
+
for (int j = 0; j <= 4 ; j++) {
|
219
|
+
assertEquals("_c" + j, record[j]);
|
220
|
+
}
|
221
|
+
}
|
222
|
+
else {
|
223
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
224
|
+
assertEquals("true", record[0]);
|
225
|
+
assertEquals("2", record[1]);
|
226
|
+
assertEquals("3.0", record[2]);
|
227
|
+
assertEquals("45", record[3]);
|
228
|
+
assertEquals("1970-01-01 00:00:00.678000 +0000", record[4]);
|
229
|
+
}
|
230
|
+
}
|
231
|
+
}
|
232
|
+
catch (IOException e) {
|
233
|
+
logger.debug(e.getMessage(), e);
|
234
|
+
}
|
235
|
+
}
|
236
|
+
|
237
|
+
@Test
|
238
|
+
public void testConfigValuesIncludingDefault()
|
239
|
+
{
|
240
|
+
// setting embulk config
|
241
|
+
final String pathPrefix = "/test/testUserPassword";
|
242
|
+
String configYaml = "" +
|
243
|
+
"type: sftp\n" +
|
244
|
+
"host: " + HOST + "\n" +
|
245
|
+
"user: " + USERNAME + "\n" +
|
246
|
+
"path_prefix: " + pathPrefix + "\n" +
|
247
|
+
"file_ext: txt\n" +
|
248
|
+
"formatter:\n" +
|
249
|
+
" type: csv\n" +
|
250
|
+
" newline: CRLF\n" +
|
251
|
+
" newline_in_field: LF\n" +
|
252
|
+
" header_line: true\n" +
|
253
|
+
" charset: UTF-8\n" +
|
254
|
+
" quote_policy: NONE\n" +
|
255
|
+
" quote: \"\\\"\"\n" +
|
256
|
+
" escape: \"\\\\\"\n" +
|
257
|
+
" null_string: \"\"\n" +
|
258
|
+
" default_timezone: 'UTC'";
|
259
|
+
|
260
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
261
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
262
|
+
|
263
|
+
assertEquals(HOST, task.getHost());
|
264
|
+
assertEquals(22, task.getPort());
|
265
|
+
assertEquals(USERNAME, task.getUser());
|
266
|
+
assertEquals(Optional.absent(), task.getPassword());
|
267
|
+
assertEquals(Optional.absent(), task.getSecretKeyFilePath());
|
268
|
+
assertEquals("", task.getSecretKeyPassphrase());
|
269
|
+
assertEquals(true, task.getUserDirIsRoot());
|
270
|
+
assertEquals(600, task.getSftpConnectionTimeout());
|
271
|
+
assertEquals(5, task.getMaxConnectionRetry());
|
272
|
+
assertEquals(pathPrefix, task.getPathPrefix());
|
273
|
+
assertEquals("txt", task.getFileNameExtension());
|
274
|
+
assertEquals("%03d.%02d.", task.getSequenceFormat());
|
275
|
+
}
|
276
|
+
|
277
|
+
// Cases
|
278
|
+
// login(all cases needs host + port)
|
279
|
+
// user + password
|
280
|
+
// user + secret_key_file + secret_key_passphrase
|
281
|
+
// put files
|
282
|
+
// user_directory_is_root
|
283
|
+
// not user_directory_is_root
|
284
|
+
// timeout
|
285
|
+
// 0 second
|
286
|
+
|
287
|
+
|
288
|
+
@Test
|
289
|
+
public void testUserPasswordAndPutToUserDirectoryRoot()
|
290
|
+
{
|
291
|
+
// setting embulk config
|
292
|
+
final String pathPrefix = "/test/testUserPassword";
|
293
|
+
String configYaml = "" +
|
294
|
+
"type: sftp\n" +
|
295
|
+
"host: " + HOST + "\n" +
|
296
|
+
"port: " + PORT + "\n" +
|
297
|
+
"user: " + USERNAME + "\n" +
|
298
|
+
"password: " + PASSWORD + "\n" +
|
299
|
+
"path_prefix: " + pathPrefix + "\n" +
|
300
|
+
"file_ext: txt\n" +
|
301
|
+
"formatter:\n" +
|
302
|
+
" type: csv\n" +
|
303
|
+
" newline: CRLF\n" +
|
304
|
+
" newline_in_field: LF\n" +
|
305
|
+
" header_line: true\n" +
|
306
|
+
" charset: UTF-8\n" +
|
307
|
+
" quote_policy: NONE\n" +
|
308
|
+
" quote: \"\\\"\"\n" +
|
309
|
+
" escape: \"\\\\\"\n" +
|
310
|
+
" null_string: \"\"\n" +
|
311
|
+
" default_timezone: 'UTC'";
|
312
|
+
|
313
|
+
// runner.transaction -> ...
|
314
|
+
run(configYaml, Optional.<Integer>absent());
|
315
|
+
|
316
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
317
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
318
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
319
|
+
testFolder.getRoot().getAbsolutePath(),
|
320
|
+
pathPrefix));
|
321
|
+
|
322
|
+
}
|
323
|
+
|
324
|
+
@Test
|
325
|
+
public void testUserSecretKeyFileAndPutToRootDirectory()
|
326
|
+
{
|
327
|
+
// setting embulk config
|
328
|
+
final String pathPrefix = "/test/testUserPassword";
|
329
|
+
String configYaml = "" +
|
330
|
+
"type: sftp\n" +
|
331
|
+
"host: " + HOST + "\n" +
|
332
|
+
"port: " + PORT + "\n" +
|
333
|
+
"user: " + USERNAME + "\n" +
|
334
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
335
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
336
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
337
|
+
"file_ext: txt\n" +
|
338
|
+
"formatter:\n" +
|
339
|
+
" type: csv\n" +
|
340
|
+
" newline: CRLF\n" +
|
341
|
+
" newline_in_field: LF\n" +
|
342
|
+
" header_line: true\n" +
|
343
|
+
" charset: UTF-8\n" +
|
344
|
+
" quote_policy: NONE\n" +
|
345
|
+
" quote: \"\\\"\"\n" +
|
346
|
+
" escape: \"\\\\\"\n" +
|
347
|
+
" null_string: \"\"\n" +
|
348
|
+
" default_timezone: 'UTC'";
|
349
|
+
|
350
|
+
// runner.transaction -> ...
|
351
|
+
run(configYaml, Optional.<Integer>absent());
|
352
|
+
|
353
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
354
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
355
|
+
|
356
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
357
|
+
testFolder.getRoot().getAbsolutePath(),
|
358
|
+
pathPrefix));
|
359
|
+
}
|
360
|
+
|
361
|
+
@Test
|
362
|
+
public void testTimeout()
|
363
|
+
{
|
364
|
+
// setting embulk config
|
365
|
+
final String pathPrefix = "/test/testUserPassword";
|
366
|
+
String configYaml = "" +
|
367
|
+
"type: sftp\n" +
|
368
|
+
"host: " + HOST + "\n" +
|
369
|
+
"port: " + PORT + "\n" +
|
370
|
+
"user: " + USERNAME + "\n" +
|
371
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
372
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
373
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
374
|
+
"timeout: 1\n" +
|
375
|
+
"file_ext: txt\n" +
|
376
|
+
"formatter:\n" +
|
377
|
+
" type: csv\n" +
|
378
|
+
" newline: CRLF\n" +
|
379
|
+
" newline_in_field: LF\n" +
|
380
|
+
" header_line: true\n" +
|
381
|
+
" charset: UTF-8\n" +
|
382
|
+
" quote_policy: NONE\n" +
|
383
|
+
" quote: \"\\\"\"\n" +
|
384
|
+
" escape: \"\\\\\"\n" +
|
385
|
+
" null_string: \"\"\n" +
|
386
|
+
" default_timezone: 'UTC'";
|
387
|
+
|
388
|
+
// exception
|
389
|
+
exception.expect(RuntimeException.class);
|
390
|
+
exception.expectCause(CoreMatchers.<Throwable>instanceOf(FileSystemException.class));
|
391
|
+
exception.expectMessage("Could not connect to SFTP server");
|
392
|
+
|
393
|
+
// runner.transaction -> ...
|
394
|
+
run(configYaml, Optional.of(60)); // sleep 1 minute while processing
|
395
|
+
}
|
5
396
|
}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
-----BEGIN RSA PRIVATE KEY-----
|
2
|
+
MIIEpAIBAAKCAQEAuRVR3XQerqx0qAP6cR3M5CvgcLoxn2UxJmlW355zyVIxHRHo
|
3
|
+
EdwQ+hNhXgSWoiZQw3fJny1GY999lCRePMETLehtbCwJlg2ju/6WhsGnzQbqpBiN
|
4
|
+
TfxQKUH+ldc/q/WApgcn2M1z9Cw05x5j0PffjMDCnxa2luRqFP4oNfEIe2460uRp
|
5
|
+
lK4gHRs7ixCRg9nIJdzmml8hr0RMCxFiMWCuwEDKh5OVW9IQmQm5XIFsiDBWAA35
|
6
|
+
BW4dq7jWD+/3/SdPQf40Yl0I/wy0XaVUbaLNt6WYn6Hc2BfcI5gVY3/4v7VjJ7ha
|
7
|
+
TBz4JocwE+SnJwk6/v/x3NCNBWSAHsX+r6mppQIDAQABAoIBAAY2iQD+HOmQMHpT
|
8
|
+
Z9zUO0SNlIM0POY052l6r8fh8dEYST3IlBnFYWyDqgkV7liTILLvGn2Vb396ySRn
|
9
|
+
9CHl+VbZLpMrB0daPdzZtwPfDFr9mHLZwCSWC7rW1E0DZMfI/gCUzOmiO/o+YahA
|
10
|
+
hOlG6OI4tk/Q22BFWqpJvd/Rr481HW1O2z+NS/jjb74UYXCHoi5k81rQsuFVKUei
|
11
|
+
JAAVHIu9ZQtlvr9hBj+9RLD0vHhKratW9gU62QJNPbe52zQyx51iDyPKhqpDr3N/
|
12
|
+
1LEzJh0ffmGznokOYo76tDe6gHe30qLWKoK0G/Xq4CQl6Xir8BIFZDifexnfGDF5
|
13
|
+
0UqGcQECgYEA6gyowwV+Wp6zsNqpNAEWLvN+maFau1lezAq3kCLmzpi16IBJ2FiT
|
14
|
+
N2cdsmN8jGK7GBLHcRr/woe0tNaPY3UInwvoycTqEnXpwV/G6kiCZopAieCccbeN
|
15
|
+
bBgxzYvwLvUAZjrzaicSPkcLy6v4mqkN+uAJoIQPGVBKyekbANKL12kCgYEAynEF
|
16
|
+
l9mYjdmzzYnBuCsiPFXYC9YaFC3c0V0LM4sWqSG7Muzl4accrkjJEWta0Zj9ZFMi
|
17
|
+
/4PuGHJqhwvP8seoBOexSGvqw7WL8TmH0VsjRpBZepewrZvHdQF6Y3x7SRFsGmeO
|
18
|
+
UrhJQJ50K7vuMAgOtpV7pG28XTVPdtdpgYwolN0CgYEAx6PwuBmRg/7nS63YspFS
|
19
|
+
uM3eaXFnEXPUFrv1hC8hLYy5bg8v7ijvukxs2mNeM+lrqccq9GGRU/+ptq9mM/H6
|
20
|
+
XqAr+x/KgeCuedIWvHUJNiorjvBWwfFN4qxonuoFACjgJFOYmi2ltBWqDBsuvc4C
|
21
|
+
LMK5s3+x2WfRw5ThAbyWqDECgYEAsrMgB9IZmyXQVSBEM4lH2FDCvHgLX0K4yMtE
|
22
|
+
RwSBRV00PRjnN9DnPWWcr9a/PYO6MeQ9YaYY99WP+ZPQxXO++k13rgEtJANO++oX
|
23
|
+
3ROBSRnFErVyTV3UrJFYcxeAUJijf49oD9cDOppk2/IxqI6ERmIf2Z7Qhsq3UaWH
|
24
|
+
ScbqT7UCgYAPQYk6uCOdBo4ad+ygf7/rI5bIlh+EufnKQB+AesMFR+4jpceQJtAY
|
25
|
+
Nvx7vg5qEWD7R9zYqY6tyLNjqbR0UV0KQYHQTH/oaqvwwJ+BgEwA/sU5zTLIsal7
|
26
|
+
kHkefd74obt+FlBSB5rWri/uEoOA8oPxEDfsvlrMAnuA7ocVZpyY9A==
|
27
|
+
-----END RSA PRIVATE KEY-----
|
@@ -0,0 +1 @@
|
|
1
|
+
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5FVHddB6urHSoA/pxHczkK+BwujGfZTEmaVbfnnPJUjEdEegR3BD6E2FeBJaiJlDDd8mfLUZj332UJF48wRMt6G1sLAmWDaO7/paGwafNBuqkGI1N/FApQf6V1z+r9YCmByfYzXP0LDTnHmPQ99+MwMKfFraW5GoU/ig18Qh7bjrS5GmUriAdGzuLEJGD2cgl3OaaXyGvREwLEWIxYK7AQMqHk5Vb0hCZCblcgWyIMFYADfkFbh2ruNYP7/f9J09B/jRiXQj/DLRdpVRtos23pZifodzYF9wjmBVjf/i/tWMnuFpMHPgmhzAT5KcnCTr+//Hc0I0FZIAexf6vqaml takahiro.nakayama@o-05796-mac.local
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-sftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -59,9 +60,11 @@ files:
|
|
59
60
|
- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
|
60
61
|
- src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
|
61
62
|
- src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
|
63
|
+
- src/test/resources/id_rsa
|
64
|
+
- src/test/resources/id_rsa.pub
|
62
65
|
- classpath/commons-logging-1.2.jar
|
63
66
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
64
|
-
- classpath/embulk-output-sftp-0.0.
|
67
|
+
- classpath/embulk-output-sftp-0.0.3.jar
|
65
68
|
- classpath/jsch-0.1.53.jar
|
66
69
|
homepage: https://github.com/civitaspo/embulk-output-sftp
|
67
70
|
licenses:
|
Binary file
|