embulk-output-sftp 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +9 -0
- data/README.md +2 -0
- data/build.gradle +15 -3
- data/classpath/embulk-output-sftp-0.0.3.jar +0 -0
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +24 -2
- data/src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java +4 -0
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +391 -0
- data/src/test/resources/id_rsa +27 -0
- data/src/test/resources/id_rsa.pub +1 -0
- metadata +6 -3
- data/classpath/embulk-output-sftp-0.0.2.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 429f131e8ee696e9543c77900ba06e550b7bf9db
|
4
|
+
data.tar.gz: e43a2290c0d7e8b18bee5af81656f688d94e5dd4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b126f0b957da348b95622b331e83b62238fccc5ed0768aefa8f626eb895335eee4ed5c1bdb5420655a64fd4ba1cdc43a9c902ec28ba70a09f9d56527c8b5581f
|
7
|
+
data.tar.gz: c41dd77673baa74ff9aad663ac28ebe71ffae50cafcf921a3970bbbdc57ec5b5e4f989fb571fdf680b5724b6c0482316908ad5be295ee1db95f7021a2edee828
|
data/.travis.yml
ADDED
data/README.md
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
# Sftp file output plugin for Embulk
|
2
|
+
[](https://travis-ci.org/civitaspo/embulk-output-sftp)
|
3
|
+
[](https://coveralls.io/github/civitaspo/embulk-output-sftp?branch=master)
|
2
4
|
|
3
5
|
Stores files on a SFTP Server
|
4
6
|
|
data/build.gradle
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
plugins {
|
2
2
|
id "com.jfrog.bintray" version "1.1"
|
3
3
|
id "com.github.jruby-gradle.base" version "0.1.5"
|
4
|
+
id "com.github.kt3k.coveralls" version "2.4.0"
|
5
|
+
id "jacoco"
|
4
6
|
id "java"
|
5
7
|
}
|
6
8
|
import com.github.jrubygradle.JRubyExec
|
@@ -12,17 +14,27 @@ configurations {
|
|
12
14
|
provided
|
13
15
|
}
|
14
16
|
|
15
|
-
version = "0.0.
|
17
|
+
version = "0.0.3"
|
16
18
|
sourceCompatibility = 1.7
|
17
19
|
targetCompatibility = 1.7
|
18
20
|
|
19
21
|
dependencies {
|
20
|
-
compile "org.embulk:embulk-core:0.7
|
21
|
-
provided "org.embulk:embulk-core:0.7
|
22
|
+
compile "org.embulk:embulk-core:0.7.+"
|
23
|
+
provided "org.embulk:embulk-core:0.7.+"
|
22
24
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
23
25
|
compile "org.apache.commons:commons-vfs2:2.+"
|
24
26
|
compile "com.jcraft:jsch:0.1.53"
|
25
27
|
testCompile "junit:junit:4.+"
|
28
|
+
testCompile "org.embulk:embulk-core:0.7.+:tests"
|
29
|
+
testCompile "org.embulk:embulk-standards:0.7.+"
|
30
|
+
testCompile "org.apache.sshd:apache-sshd:1.+"
|
31
|
+
}
|
32
|
+
|
33
|
+
jacocoTestReport {
|
34
|
+
reports {
|
35
|
+
xml.enabled = true // coveralls plugin depends on xml format report
|
36
|
+
html.enabled = true
|
37
|
+
}
|
26
38
|
}
|
27
39
|
|
28
40
|
task classpath(type: Copy, dependsOn: ["jar"]) {
|
Binary file
|
@@ -4,7 +4,6 @@ import com.google.common.base.Throwables;
|
|
4
4
|
import org.apache.commons.vfs2.FileObject;
|
5
5
|
import org.apache.commons.vfs2.FileSystemException;
|
6
6
|
import org.apache.commons.vfs2.FileSystemOptions;
|
7
|
-
import org.apache.commons.vfs2.Selectors;
|
8
7
|
import org.apache.commons.vfs2.impl.StandardFileSystemManager;
|
9
8
|
import org.apache.commons.vfs2.provider.sftp.IdentityInfo;
|
10
9
|
import org.apache.commons.vfs2.provider.sftp.SftpFileSystemConfigBuilder;
|
@@ -35,6 +34,7 @@ public class SftpFileOutput
|
|
35
34
|
private final String userInfo;
|
36
35
|
private final String host;
|
37
36
|
private final int port;
|
37
|
+
private final int maxConnectionRetry;
|
38
38
|
private final String pathPrefix;
|
39
39
|
private final String sequenceFormat;
|
40
40
|
private final String fileNameExtension;
|
@@ -82,6 +82,7 @@ public class SftpFileOutput
|
|
82
82
|
if (task.getSecretKeyFilePath().isPresent()) {
|
83
83
|
IdentityInfo identityInfo = new IdentityInfo(new File((task.getSecretKeyFilePath().get())), task.getSecretKeyPassphrase().getBytes());
|
84
84
|
SftpFileSystemConfigBuilder.getInstance().setIdentityInfo(fsOptions, identityInfo);
|
85
|
+
logger.info("set identity: {}", task.getSecretKeyFilePath().get());
|
85
86
|
}
|
86
87
|
}
|
87
88
|
catch (FileSystemException e) {
|
@@ -99,6 +100,7 @@ public class SftpFileOutput
|
|
99
100
|
this.fsOptions = initializeFsOptions(task);
|
100
101
|
this.host = task.getHost();
|
101
102
|
this.port = task.getPort();
|
103
|
+
this.maxConnectionRetry = task.getMaxConnectionRetry();
|
102
104
|
this.pathPrefix = task.getPathPrefix();
|
103
105
|
this.sequenceFormat = task.getSequenceFormat();
|
104
106
|
this.fileNameExtension = task.getFileNameExtension();
|
@@ -204,6 +206,26 @@ public class SftpFileOutput
|
|
204
206
|
private FileObject newSftpFile(URI sftpUri)
|
205
207
|
throws FileSystemException
|
206
208
|
{
|
207
|
-
|
209
|
+
int count = 0;
|
210
|
+
while (true) {
|
211
|
+
try {
|
212
|
+
return manager.resolveFile(sftpUri.toString(), fsOptions);
|
213
|
+
}
|
214
|
+
catch (FileSystemException e) {
|
215
|
+
if (++count == maxConnectionRetry) {
|
216
|
+
throw e;
|
217
|
+
}
|
218
|
+
logger.warn("failed to connect sftp server: " + e.getMessage(), e);
|
219
|
+
|
220
|
+
try {
|
221
|
+
Thread.sleep(count * 1000); // milliseconds
|
222
|
+
}
|
223
|
+
catch (InterruptedException e1) {
|
224
|
+
// Ignore this exception
|
225
|
+
logger.warn(e.getMessage(), e);
|
226
|
+
}
|
227
|
+
logger.warn("retry to connect sftp server: " + count + " times");
|
228
|
+
}
|
229
|
+
}
|
208
230
|
}
|
209
231
|
}
|
@@ -53,6 +53,10 @@ public class SftpFileOutputPlugin
|
|
53
53
|
@ConfigDefault("600") // 10 minutes
|
54
54
|
public int getSftpConnectionTimeout();
|
55
55
|
|
56
|
+
@Config("max_connection_retry")
|
57
|
+
@ConfigDefault("5") // 5 times retry to connect sftp server if failed.
|
58
|
+
public int getMaxConnectionRetry();
|
59
|
+
|
56
60
|
@Config("path_prefix")
|
57
61
|
public String getPathPrefix();
|
58
62
|
|
@@ -1,5 +1,396 @@
|
|
1
1
|
package org.embulk.output.sftp;
|
2
2
|
|
3
|
+
import com.google.common.base.Charsets;
|
4
|
+
import com.google.common.base.Optional;
|
5
|
+
import com.google.common.base.Splitter;
|
6
|
+
import com.google.common.base.Throwables;
|
7
|
+
import com.google.common.collect.Lists;
|
8
|
+
import com.google.common.io.Resources;
|
9
|
+
import org.apache.commons.vfs2.FileSystemException;
|
10
|
+
import org.apache.sshd.common.NamedFactory;
|
11
|
+
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
|
12
|
+
import org.apache.sshd.server.Command;
|
13
|
+
import org.apache.sshd.server.SshServer;
|
14
|
+
import org.apache.sshd.server.auth.password.PasswordAuthenticator;
|
15
|
+
import org.apache.sshd.server.auth.pubkey.PublickeyAuthenticator;
|
16
|
+
import org.apache.sshd.server.command.ScpCommandFactory;
|
17
|
+
import org.apache.sshd.server.keyprovider.SimpleGeneratorHostKeyProvider;
|
18
|
+
import org.apache.sshd.server.session.ServerSession;
|
19
|
+
import org.apache.sshd.server.subsystem.sftp.SftpSubsystemFactory;
|
20
|
+
import org.embulk.EmbulkTestRuntime;
|
21
|
+
import org.embulk.config.ConfigLoader;
|
22
|
+
import org.embulk.config.ConfigSource;
|
23
|
+
import org.embulk.config.TaskReport;
|
24
|
+
import org.embulk.config.TaskSource;
|
25
|
+
import org.embulk.output.sftp.SftpFileOutputPlugin.PluginTask;
|
26
|
+
import org.embulk.spi.Exec;
|
27
|
+
import org.embulk.spi.FileOutputRunner;
|
28
|
+
import org.embulk.spi.OutputPlugin.Control;
|
29
|
+
import org.embulk.spi.Page;
|
30
|
+
import org.embulk.spi.PageTestUtils;
|
31
|
+
import org.embulk.spi.Schema;
|
32
|
+
import org.embulk.spi.TransactionalPageOutput;
|
33
|
+
import org.embulk.spi.time.Timestamp;
|
34
|
+
import org.hamcrest.CoreMatchers;
|
35
|
+
import org.hamcrest.Matcher;
|
36
|
+
import org.junit.After;
|
37
|
+
import org.junit.Before;
|
38
|
+
import org.junit.Rule;
|
39
|
+
import org.junit.Test;
|
40
|
+
import org.junit.rules.ExpectedException;
|
41
|
+
import org.junit.rules.TemporaryFolder;
|
42
|
+
import org.slf4j.Logger;
|
43
|
+
|
44
|
+
import java.io.File;
|
45
|
+
import java.io.IOException;
|
46
|
+
import java.nio.file.DirectoryStream;
|
47
|
+
import java.nio.file.FileSystems;
|
48
|
+
import java.nio.file.Files;
|
49
|
+
import java.nio.file.Path;
|
50
|
+
import java.nio.file.PathMatcher;
|
51
|
+
import java.nio.file.Paths;
|
52
|
+
import java.security.PublicKey;
|
53
|
+
import java.util.Arrays;
|
54
|
+
import java.util.Collections;
|
55
|
+
import java.util.List;
|
56
|
+
|
57
|
+
import static com.google.common.io.Files.readLines;
|
58
|
+
import static org.embulk.spi.type.Types.*;
|
59
|
+
import static org.hamcrest.CoreMatchers.containsString;
|
60
|
+
import static org.hamcrest.CoreMatchers.hasItem;
|
61
|
+
import static org.hamcrest.CoreMatchers.instanceOf;
|
62
|
+
import static org.hamcrest.MatcherAssert.*;
|
63
|
+
import static org.hamcrest.core.Is.is;
|
64
|
+
import static org.junit.Assert.assertEquals;
|
65
|
+
//import static org.hamcrest.Matchers.*;
|
66
|
+
|
3
67
|
public class TestSftpFileOutputPlugin
|
4
68
|
{
|
69
|
+
@Rule
|
70
|
+
public EmbulkTestRuntime runtime = new EmbulkTestRuntime();
|
71
|
+
|
72
|
+
@Rule
|
73
|
+
public ExpectedException exception = ExpectedException.none();
|
74
|
+
|
75
|
+
@Rule
|
76
|
+
public TemporaryFolder testFolder = new TemporaryFolder();
|
77
|
+
|
78
|
+
private Logger logger = runtime.getExec().getLogger(TestSftpFileOutputPlugin.class);
|
79
|
+
private FileOutputRunner runner;
|
80
|
+
private SshServer sshServer;
|
81
|
+
private static final String HOST = "127.0.0.1";
|
82
|
+
private static final int PORT = 20022;
|
83
|
+
private static final String USERNAME = "username";
|
84
|
+
private static final String PASSWORD = "password";
|
85
|
+
private static final String SECRET_KEY_FILE = Resources.getResource("id_rsa").getPath();
|
86
|
+
private static final String SECRET_KEY_PASSPHRASE = "SECRET_KEY_PASSPHRASE";
|
87
|
+
private static final Schema SCHEMA = new Schema.Builder()
|
88
|
+
.add("_c0", BOOLEAN)
|
89
|
+
.add("_c1", LONG)
|
90
|
+
.add("_c2", DOUBLE)
|
91
|
+
.add("_c3", STRING)
|
92
|
+
.add("_c4", TIMESTAMP)
|
93
|
+
.build();
|
94
|
+
|
95
|
+
@Before
|
96
|
+
public void createResources()
|
97
|
+
throws IOException
|
98
|
+
{
|
99
|
+
// setup the plugin
|
100
|
+
SftpFileOutputPlugin sftpFileOutputPlugin = new SftpFileOutputPlugin();
|
101
|
+
runner = new FileOutputRunner(sftpFileOutputPlugin);
|
102
|
+
|
103
|
+
// setup a mock sftp server
|
104
|
+
sshServer = SshServer.setUpDefaultServer();
|
105
|
+
VirtualFileSystemFactory fsFactory = new VirtualFileSystemFactory();
|
106
|
+
fsFactory.setUserHomeDir(USERNAME, testFolder.getRoot().getAbsolutePath());
|
107
|
+
sshServer.setFileSystemFactory(fsFactory);
|
108
|
+
sshServer.setHost(HOST);
|
109
|
+
sshServer.setPort(PORT);
|
110
|
+
sshServer.setSubsystemFactories(Collections.<NamedFactory<Command>>singletonList(new SftpSubsystemFactory()));
|
111
|
+
sshServer.setCommandFactory(new ScpCommandFactory());
|
112
|
+
sshServer.setKeyPairProvider(new SimpleGeneratorHostKeyProvider());
|
113
|
+
sshServer.setPasswordAuthenticator(new PasswordAuthenticator()
|
114
|
+
{
|
115
|
+
@Override
|
116
|
+
public boolean authenticate(final String username, final String password, final ServerSession session)
|
117
|
+
{
|
118
|
+
return USERNAME.contentEquals(username) && PASSWORD.contentEquals(password);
|
119
|
+
}
|
120
|
+
});
|
121
|
+
sshServer.setPublickeyAuthenticator(new PublickeyAuthenticator()
|
122
|
+
{
|
123
|
+
@Override
|
124
|
+
public boolean authenticate(String username, PublicKey key, ServerSession session)
|
125
|
+
{
|
126
|
+
return true;
|
127
|
+
}
|
128
|
+
});
|
129
|
+
|
130
|
+
try {
|
131
|
+
sshServer.start();
|
132
|
+
}
|
133
|
+
catch (IOException e) {
|
134
|
+
logger.debug(e.getMessage(), e);
|
135
|
+
}
|
136
|
+
}
|
137
|
+
|
138
|
+
@After
|
139
|
+
public void cleanup() throws InterruptedException {
|
140
|
+
try {
|
141
|
+
sshServer.stop(true);
|
142
|
+
}
|
143
|
+
catch (Exception e) {
|
144
|
+
logger.debug(e.getMessage(), e);
|
145
|
+
}
|
146
|
+
}
|
147
|
+
|
148
|
+
private ConfigSource getConfigFromYaml(String yaml)
|
149
|
+
{
|
150
|
+
ConfigLoader loader = new ConfigLoader(Exec.getModelManager());
|
151
|
+
return loader.fromYamlString(yaml);
|
152
|
+
}
|
153
|
+
|
154
|
+
private List<String> lsR(List<String> fileNames, Path dir) {
|
155
|
+
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir)) {
|
156
|
+
for (Path path : stream) {
|
157
|
+
if(path.toFile().isDirectory()) {
|
158
|
+
lsR(fileNames, path);
|
159
|
+
} else {
|
160
|
+
fileNames.add(path.toAbsolutePath().toString());
|
161
|
+
}
|
162
|
+
}
|
163
|
+
}
|
164
|
+
catch(IOException e) {
|
165
|
+
logger.debug(e.getMessage(), e);
|
166
|
+
}
|
167
|
+
return fileNames;
|
168
|
+
}
|
169
|
+
|
170
|
+
private void run(String configYaml, final Optional<Integer> sleep)
|
171
|
+
{
|
172
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
173
|
+
runner.transaction(config, SCHEMA, 1, new Control()
|
174
|
+
{
|
175
|
+
@Override
|
176
|
+
public List<TaskReport> run(TaskSource taskSource)
|
177
|
+
{
|
178
|
+
TransactionalPageOutput pageOutput = runner.open(taskSource, SCHEMA, 1);
|
179
|
+
boolean committed = false;
|
180
|
+
try {
|
181
|
+
// Result:
|
182
|
+
// _c0,_c1,_c2,_c3,_c4
|
183
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
184
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
185
|
+
for (Page page : PageTestUtils.buildPage(runtime.getBufferAllocator(), SCHEMA, true, 2L, 3.0D, "45",
|
186
|
+
Timestamp.ofEpochMilli(678L), true, 2L, 3.0D, "45",
|
187
|
+
Timestamp.ofEpochMilli(678L))) {
|
188
|
+
pageOutput.add(page);
|
189
|
+
if (sleep.isPresent()) {
|
190
|
+
Thread.sleep(sleep.get() * 1000);
|
191
|
+
}
|
192
|
+
}
|
193
|
+
pageOutput.commit();
|
194
|
+
committed = true;
|
195
|
+
}
|
196
|
+
catch (InterruptedException e) {
|
197
|
+
logger.debug(e.getMessage(), e);
|
198
|
+
}
|
199
|
+
finally {
|
200
|
+
if (!committed) {
|
201
|
+
pageOutput.abort();
|
202
|
+
}
|
203
|
+
pageOutput.close();
|
204
|
+
}
|
205
|
+
return Lists.newArrayList();
|
206
|
+
}
|
207
|
+
});
|
208
|
+
}
|
209
|
+
|
210
|
+
private void assertRecordsInFile(String filePath)
|
211
|
+
{
|
212
|
+
try {
|
213
|
+
List<String> lines = readLines(new File(filePath),
|
214
|
+
Charsets.UTF_8);
|
215
|
+
for (int i = 0; i < lines.size(); i++) {
|
216
|
+
String[] record = lines.get(i).split(",");
|
217
|
+
if (i == 0) {
|
218
|
+
for (int j = 0; j <= 4 ; j++) {
|
219
|
+
assertEquals("_c" + j, record[j]);
|
220
|
+
}
|
221
|
+
}
|
222
|
+
else {
|
223
|
+
// true,2,3.0,45,1970-01-01 00:00:00.678000 +0000
|
224
|
+
assertEquals("true", record[0]);
|
225
|
+
assertEquals("2", record[1]);
|
226
|
+
assertEquals("3.0", record[2]);
|
227
|
+
assertEquals("45", record[3]);
|
228
|
+
assertEquals("1970-01-01 00:00:00.678000 +0000", record[4]);
|
229
|
+
}
|
230
|
+
}
|
231
|
+
}
|
232
|
+
catch (IOException e) {
|
233
|
+
logger.debug(e.getMessage(), e);
|
234
|
+
}
|
235
|
+
}
|
236
|
+
|
237
|
+
@Test
|
238
|
+
public void testConfigValuesIncludingDefault()
|
239
|
+
{
|
240
|
+
// setting embulk config
|
241
|
+
final String pathPrefix = "/test/testUserPassword";
|
242
|
+
String configYaml = "" +
|
243
|
+
"type: sftp\n" +
|
244
|
+
"host: " + HOST + "\n" +
|
245
|
+
"user: " + USERNAME + "\n" +
|
246
|
+
"path_prefix: " + pathPrefix + "\n" +
|
247
|
+
"file_ext: txt\n" +
|
248
|
+
"formatter:\n" +
|
249
|
+
" type: csv\n" +
|
250
|
+
" newline: CRLF\n" +
|
251
|
+
" newline_in_field: LF\n" +
|
252
|
+
" header_line: true\n" +
|
253
|
+
" charset: UTF-8\n" +
|
254
|
+
" quote_policy: NONE\n" +
|
255
|
+
" quote: \"\\\"\"\n" +
|
256
|
+
" escape: \"\\\\\"\n" +
|
257
|
+
" null_string: \"\"\n" +
|
258
|
+
" default_timezone: 'UTC'";
|
259
|
+
|
260
|
+
ConfigSource config = getConfigFromYaml(configYaml);
|
261
|
+
PluginTask task = config.loadConfig(PluginTask.class);
|
262
|
+
|
263
|
+
assertEquals(HOST, task.getHost());
|
264
|
+
assertEquals(22, task.getPort());
|
265
|
+
assertEquals(USERNAME, task.getUser());
|
266
|
+
assertEquals(Optional.absent(), task.getPassword());
|
267
|
+
assertEquals(Optional.absent(), task.getSecretKeyFilePath());
|
268
|
+
assertEquals("", task.getSecretKeyPassphrase());
|
269
|
+
assertEquals(true, task.getUserDirIsRoot());
|
270
|
+
assertEquals(600, task.getSftpConnectionTimeout());
|
271
|
+
assertEquals(5, task.getMaxConnectionRetry());
|
272
|
+
assertEquals(pathPrefix, task.getPathPrefix());
|
273
|
+
assertEquals("txt", task.getFileNameExtension());
|
274
|
+
assertEquals("%03d.%02d.", task.getSequenceFormat());
|
275
|
+
}
|
276
|
+
|
277
|
+
// Cases
|
278
|
+
// login(all cases needs host + port)
|
279
|
+
// user + password
|
280
|
+
// user + secret_key_file + secret_key_passphrase
|
281
|
+
// put files
|
282
|
+
// user_directory_is_root
|
283
|
+
// not user_directory_is_root
|
284
|
+
// timeout
|
285
|
+
// 0 second
|
286
|
+
|
287
|
+
|
288
|
+
@Test
|
289
|
+
public void testUserPasswordAndPutToUserDirectoryRoot()
|
290
|
+
{
|
291
|
+
// setting embulk config
|
292
|
+
final String pathPrefix = "/test/testUserPassword";
|
293
|
+
String configYaml = "" +
|
294
|
+
"type: sftp\n" +
|
295
|
+
"host: " + HOST + "\n" +
|
296
|
+
"port: " + PORT + "\n" +
|
297
|
+
"user: " + USERNAME + "\n" +
|
298
|
+
"password: " + PASSWORD + "\n" +
|
299
|
+
"path_prefix: " + pathPrefix + "\n" +
|
300
|
+
"file_ext: txt\n" +
|
301
|
+
"formatter:\n" +
|
302
|
+
" type: csv\n" +
|
303
|
+
" newline: CRLF\n" +
|
304
|
+
" newline_in_field: LF\n" +
|
305
|
+
" header_line: true\n" +
|
306
|
+
" charset: UTF-8\n" +
|
307
|
+
" quote_policy: NONE\n" +
|
308
|
+
" quote: \"\\\"\"\n" +
|
309
|
+
" escape: \"\\\\\"\n" +
|
310
|
+
" null_string: \"\"\n" +
|
311
|
+
" default_timezone: 'UTC'";
|
312
|
+
|
313
|
+
// runner.transaction -> ...
|
314
|
+
run(configYaml, Optional.<Integer>absent());
|
315
|
+
|
316
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
317
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
318
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
319
|
+
testFolder.getRoot().getAbsolutePath(),
|
320
|
+
pathPrefix));
|
321
|
+
|
322
|
+
}
|
323
|
+
|
324
|
+
@Test
|
325
|
+
public void testUserSecretKeyFileAndPutToRootDirectory()
|
326
|
+
{
|
327
|
+
// setting embulk config
|
328
|
+
final String pathPrefix = "/test/testUserPassword";
|
329
|
+
String configYaml = "" +
|
330
|
+
"type: sftp\n" +
|
331
|
+
"host: " + HOST + "\n" +
|
332
|
+
"port: " + PORT + "\n" +
|
333
|
+
"user: " + USERNAME + "\n" +
|
334
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
335
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
336
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
337
|
+
"file_ext: txt\n" +
|
338
|
+
"formatter:\n" +
|
339
|
+
" type: csv\n" +
|
340
|
+
" newline: CRLF\n" +
|
341
|
+
" newline_in_field: LF\n" +
|
342
|
+
" header_line: true\n" +
|
343
|
+
" charset: UTF-8\n" +
|
344
|
+
" quote_policy: NONE\n" +
|
345
|
+
" quote: \"\\\"\"\n" +
|
346
|
+
" escape: \"\\\\\"\n" +
|
347
|
+
" null_string: \"\"\n" +
|
348
|
+
" default_timezone: 'UTC'";
|
349
|
+
|
350
|
+
// runner.transaction -> ...
|
351
|
+
run(configYaml, Optional.<Integer>absent());
|
352
|
+
|
353
|
+
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
354
|
+
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
355
|
+
|
356
|
+
assertRecordsInFile(String.format("%s/%s001.00.txt",
|
357
|
+
testFolder.getRoot().getAbsolutePath(),
|
358
|
+
pathPrefix));
|
359
|
+
}
|
360
|
+
|
361
|
+
@Test
|
362
|
+
public void testTimeout()
|
363
|
+
{
|
364
|
+
// setting embulk config
|
365
|
+
final String pathPrefix = "/test/testUserPassword";
|
366
|
+
String configYaml = "" +
|
367
|
+
"type: sftp\n" +
|
368
|
+
"host: " + HOST + "\n" +
|
369
|
+
"port: " + PORT + "\n" +
|
370
|
+
"user: " + USERNAME + "\n" +
|
371
|
+
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
372
|
+
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
373
|
+
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
374
|
+
"timeout: 1\n" +
|
375
|
+
"file_ext: txt\n" +
|
376
|
+
"formatter:\n" +
|
377
|
+
" type: csv\n" +
|
378
|
+
" newline: CRLF\n" +
|
379
|
+
" newline_in_field: LF\n" +
|
380
|
+
" header_line: true\n" +
|
381
|
+
" charset: UTF-8\n" +
|
382
|
+
" quote_policy: NONE\n" +
|
383
|
+
" quote: \"\\\"\"\n" +
|
384
|
+
" escape: \"\\\\\"\n" +
|
385
|
+
" null_string: \"\"\n" +
|
386
|
+
" default_timezone: 'UTC'";
|
387
|
+
|
388
|
+
// exception
|
389
|
+
exception.expect(RuntimeException.class);
|
390
|
+
exception.expectCause(CoreMatchers.<Throwable>instanceOf(FileSystemException.class));
|
391
|
+
exception.expectMessage("Could not connect to SFTP server");
|
392
|
+
|
393
|
+
// runner.transaction -> ...
|
394
|
+
run(configYaml, Optional.of(60)); // sleep 1 minute while processing
|
395
|
+
}
|
5
396
|
}
|
@@ -0,0 +1,27 @@
|
|
1
|
+
-----BEGIN RSA PRIVATE KEY-----
|
2
|
+
MIIEpAIBAAKCAQEAuRVR3XQerqx0qAP6cR3M5CvgcLoxn2UxJmlW355zyVIxHRHo
|
3
|
+
EdwQ+hNhXgSWoiZQw3fJny1GY999lCRePMETLehtbCwJlg2ju/6WhsGnzQbqpBiN
|
4
|
+
TfxQKUH+ldc/q/WApgcn2M1z9Cw05x5j0PffjMDCnxa2luRqFP4oNfEIe2460uRp
|
5
|
+
lK4gHRs7ixCRg9nIJdzmml8hr0RMCxFiMWCuwEDKh5OVW9IQmQm5XIFsiDBWAA35
|
6
|
+
BW4dq7jWD+/3/SdPQf40Yl0I/wy0XaVUbaLNt6WYn6Hc2BfcI5gVY3/4v7VjJ7ha
|
7
|
+
TBz4JocwE+SnJwk6/v/x3NCNBWSAHsX+r6mppQIDAQABAoIBAAY2iQD+HOmQMHpT
|
8
|
+
Z9zUO0SNlIM0POY052l6r8fh8dEYST3IlBnFYWyDqgkV7liTILLvGn2Vb396ySRn
|
9
|
+
9CHl+VbZLpMrB0daPdzZtwPfDFr9mHLZwCSWC7rW1E0DZMfI/gCUzOmiO/o+YahA
|
10
|
+
hOlG6OI4tk/Q22BFWqpJvd/Rr481HW1O2z+NS/jjb74UYXCHoi5k81rQsuFVKUei
|
11
|
+
JAAVHIu9ZQtlvr9hBj+9RLD0vHhKratW9gU62QJNPbe52zQyx51iDyPKhqpDr3N/
|
12
|
+
1LEzJh0ffmGznokOYo76tDe6gHe30qLWKoK0G/Xq4CQl6Xir8BIFZDifexnfGDF5
|
13
|
+
0UqGcQECgYEA6gyowwV+Wp6zsNqpNAEWLvN+maFau1lezAq3kCLmzpi16IBJ2FiT
|
14
|
+
N2cdsmN8jGK7GBLHcRr/woe0tNaPY3UInwvoycTqEnXpwV/G6kiCZopAieCccbeN
|
15
|
+
bBgxzYvwLvUAZjrzaicSPkcLy6v4mqkN+uAJoIQPGVBKyekbANKL12kCgYEAynEF
|
16
|
+
l9mYjdmzzYnBuCsiPFXYC9YaFC3c0V0LM4sWqSG7Muzl4accrkjJEWta0Zj9ZFMi
|
17
|
+
/4PuGHJqhwvP8seoBOexSGvqw7WL8TmH0VsjRpBZepewrZvHdQF6Y3x7SRFsGmeO
|
18
|
+
UrhJQJ50K7vuMAgOtpV7pG28XTVPdtdpgYwolN0CgYEAx6PwuBmRg/7nS63YspFS
|
19
|
+
uM3eaXFnEXPUFrv1hC8hLYy5bg8v7ijvukxs2mNeM+lrqccq9GGRU/+ptq9mM/H6
|
20
|
+
XqAr+x/KgeCuedIWvHUJNiorjvBWwfFN4qxonuoFACjgJFOYmi2ltBWqDBsuvc4C
|
21
|
+
LMK5s3+x2WfRw5ThAbyWqDECgYEAsrMgB9IZmyXQVSBEM4lH2FDCvHgLX0K4yMtE
|
22
|
+
RwSBRV00PRjnN9DnPWWcr9a/PYO6MeQ9YaYY99WP+ZPQxXO++k13rgEtJANO++oX
|
23
|
+
3ROBSRnFErVyTV3UrJFYcxeAUJijf49oD9cDOppk2/IxqI6ERmIf2Z7Qhsq3UaWH
|
24
|
+
ScbqT7UCgYAPQYk6uCOdBo4ad+ygf7/rI5bIlh+EufnKQB+AesMFR+4jpceQJtAY
|
25
|
+
Nvx7vg5qEWD7R9zYqY6tyLNjqbR0UV0KQYHQTH/oaqvwwJ+BgEwA/sU5zTLIsal7
|
26
|
+
kHkefd74obt+FlBSB5rWri/uEoOA8oPxEDfsvlrMAnuA7ocVZpyY9A==
|
27
|
+
-----END RSA PRIVATE KEY-----
|
@@ -0,0 +1 @@
|
|
1
|
+
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC5FVHddB6urHSoA/pxHczkK+BwujGfZTEmaVbfnnPJUjEdEegR3BD6E2FeBJaiJlDDd8mfLUZj332UJF48wRMt6G1sLAmWDaO7/paGwafNBuqkGI1N/FApQf6V1z+r9YCmByfYzXP0LDTnHmPQ99+MwMKfFraW5GoU/ig18Qh7bjrS5GmUriAdGzuLEJGD2cgl3OaaXyGvREwLEWIxYK7AQMqHk5Vb0hCZCblcgWyIMFYADfkFbh2ruNYP7/f9J09B/jRiXQj/DLRdpVRtos23pZifodzYF9wjmBVjf/i/tWMnuFpMHPgmhzAT5KcnCTr+//Hc0I0FZIAexf6vqaml takahiro.nakayama@o-05796-mac.local
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-sftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -46,6 +46,7 @@ extensions: []
|
|
46
46
|
extra_rdoc_files: []
|
47
47
|
files:
|
48
48
|
- .gitignore
|
49
|
+
- .travis.yml
|
49
50
|
- LICENSE.txt
|
50
51
|
- README.md
|
51
52
|
- build.gradle
|
@@ -59,9 +60,11 @@ files:
|
|
59
60
|
- src/main/java/org/embulk/output/sftp/SftpFileOutput.java
|
60
61
|
- src/main/java/org/embulk/output/sftp/SftpFileOutputPlugin.java
|
61
62
|
- src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java
|
63
|
+
- src/test/resources/id_rsa
|
64
|
+
- src/test/resources/id_rsa.pub
|
62
65
|
- classpath/commons-logging-1.2.jar
|
63
66
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
64
|
-
- classpath/embulk-output-sftp-0.0.
|
67
|
+
- classpath/embulk-output-sftp-0.0.3.jar
|
65
68
|
- classpath/jsch-0.1.53.jar
|
66
69
|
homepage: https://github.com/civitaspo/embulk-output-sftp
|
67
70
|
licenses:
|
Binary file
|