embulk-output-sftp 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +9 -0
- data/build.gradle +2 -2
- data/classpath/embulk-output-sftp-0.0.8.jar +0 -0
- data/src/main/java/org/embulk/output/sftp/SftpFileOutput.java +8 -8
- data/src/test/java/org/embulk/output/sftp/TestSftpFileOutputPlugin.java +4 -48
- metadata +13 -13
- data/classpath/embulk-output-sftp-0.0.7.jar +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ad46da79ff43c877955983ffd837db623628c9d7
|
4
|
+
data.tar.gz: 46aedede7ab91cd623f6e1d00da01a6b8498c143
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5ecdfe335690467595c9d810b86ab63d9bd0dbf2fad31d1d888fe69e3f8557d925cdea42f0e38108176fb3958c2647d17c0345914a1c5de89922da6bd72d51c5
|
7
|
+
data.tar.gz: 1834934554622086027d05dca31bedad8415206999bc159df1c12c4bb25dcd9c75c1ee7ec12ca71755de91b4746313572e19cdbfd55c0d75467c9208ef40a1c8
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,12 @@
|
|
1
|
+
0.0.8 (2016-09-26)
|
2
|
+
==================
|
3
|
+
- Fix: Use second as timetout setting instead of milli second
|
4
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/22
|
5
|
+
- Fix: Fix CI failure only with Java7
|
6
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/21
|
7
|
+
- Fix: Format code that were warned by `./gradlew checkstyle` command
|
8
|
+
- https://github.com/civitaspo/embulk-output-sftp/pull/23
|
9
|
+
|
1
10
|
0.0.7 (2016-03-22)
|
2
11
|
==================
|
3
12
|
- Fix: Plugin throws ClassNotFoundException with EmbulkEmbed
|
data/build.gradle
CHANGED
@@ -15,7 +15,7 @@ configurations {
|
|
15
15
|
provided
|
16
16
|
}
|
17
17
|
|
18
|
-
version = "0.0.
|
18
|
+
version = "0.0.8"
|
19
19
|
sourceCompatibility = 1.7
|
20
20
|
targetCompatibility = 1.7
|
21
21
|
|
@@ -23,7 +23,7 @@ dependencies {
|
|
23
23
|
compile "org.embulk:embulk-core:0.8.6"
|
24
24
|
provided "org.embulk:embulk-core:0.8.6"
|
25
25
|
// compile "YOUR_JAR_DEPENDENCY_GROUP:YOUR_JAR_DEPENDENCY_MODULE:YOUR_JAR_DEPENDENCY_VERSION"
|
26
|
-
compile "org.apache.commons:commons-vfs2:2
|
26
|
+
compile "org.apache.commons:commons-vfs2:2.1.1660580.2"
|
27
27
|
compile "com.jcraft:jsch:0.1.53"
|
28
28
|
testCompile "junit:junit:4.+"
|
29
29
|
testCompile "org.embulk:embulk-core:0.8.6:tests"
|
Binary file
|
@@ -17,7 +17,6 @@ import org.embulk.spi.TransactionalFileOutput;
|
|
17
17
|
import org.embulk.spi.unit.LocalFile;
|
18
18
|
import org.slf4j.Logger;
|
19
19
|
|
20
|
-
import java.lang.Void;
|
21
20
|
import java.io.File;
|
22
21
|
import java.io.IOException;
|
23
22
|
import java.io.OutputStream;
|
@@ -83,7 +82,7 @@ public class SftpFileOutput
|
|
83
82
|
try {
|
84
83
|
SftpFileSystemConfigBuilder builder = SftpFileSystemConfigBuilder.getInstance();
|
85
84
|
builder.setUserDirIsRoot(fsOptions, task.getUserDirIsRoot());
|
86
|
-
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout());
|
85
|
+
builder.setTimeout(fsOptions, task.getSftpConnectionTimeout() * 1000);
|
87
86
|
builder.setStrictHostKeyChecking(fsOptions, "no");
|
88
87
|
if (task.getSecretKeyFilePath().isPresent()) {
|
89
88
|
IdentityInfo identityInfo = new IdentityInfo(
|
@@ -174,7 +173,7 @@ public class SftpFileOutput
|
|
174
173
|
withConnectionRetry(retriable);
|
175
174
|
}
|
176
175
|
catch (Exception e) {
|
177
|
-
throw (IOException)e;
|
176
|
+
throw (IOException) e;
|
178
177
|
}
|
179
178
|
}
|
180
179
|
catch (IOException e) {
|
@@ -255,20 +254,21 @@ public class SftpFileOutput
|
|
255
254
|
{
|
256
255
|
/**
|
257
256
|
* Execute the operation with the given (or null) return value.
|
258
|
-
*
|
259
257
|
* @return any return value from the operation
|
260
258
|
* @throws Exception
|
261
259
|
*/
|
262
260
|
public T execute() throws Exception;
|
263
261
|
}
|
264
262
|
|
265
|
-
private <T> T withConnectionRetry(
|
263
|
+
private <T> T withConnectionRetry(final Retriable<T> op)
|
264
|
+
throws Exception
|
265
|
+
{
|
266
266
|
int count = 0;
|
267
267
|
while (true) {
|
268
268
|
try {
|
269
269
|
return op.execute();
|
270
270
|
}
|
271
|
-
catch(final Exception e) {
|
271
|
+
catch (final Exception e) {
|
272
272
|
if (++count > maxConnectionRetry) {
|
273
273
|
throw e;
|
274
274
|
}
|
@@ -309,7 +309,7 @@ public class SftpFileOutput
|
|
309
309
|
return withConnectionRetry(retriable);
|
310
310
|
}
|
311
311
|
catch (Exception e) {
|
312
|
-
throw (FileSystemException)e;
|
312
|
+
throw (FileSystemException) e;
|
313
313
|
}
|
314
314
|
}
|
315
315
|
|
@@ -326,7 +326,7 @@ public class SftpFileOutput
|
|
326
326
|
return withConnectionRetry(retriable);
|
327
327
|
}
|
328
328
|
catch (Exception e) {
|
329
|
-
throw (FileSystemException)e;
|
329
|
+
throw (FileSystemException) e;
|
330
330
|
}
|
331
331
|
}
|
332
332
|
|
@@ -4,7 +4,6 @@ import com.google.common.base.Charsets;
|
|
4
4
|
import com.google.common.base.Optional;
|
5
5
|
import com.google.common.collect.Lists;
|
6
6
|
import com.google.common.io.Resources;
|
7
|
-
import org.apache.commons.vfs2.FileSystemException;
|
8
7
|
import org.apache.sshd.common.NamedFactory;
|
9
8
|
import org.apache.sshd.common.file.virtualfs.VirtualFileSystemFactory;
|
10
9
|
import org.apache.sshd.server.Command;
|
@@ -30,7 +29,6 @@ import org.embulk.spi.PageTestUtils;
|
|
30
29
|
import org.embulk.spi.Schema;
|
31
30
|
import org.embulk.spi.TransactionalPageOutput;
|
32
31
|
import org.embulk.spi.time.Timestamp;
|
33
|
-
import org.hamcrest.CoreMatchers;
|
34
32
|
import org.junit.After;
|
35
33
|
import org.junit.Before;
|
36
34
|
import org.junit.Rule;
|
@@ -187,7 +185,7 @@ public class TestSftpFileOutputPlugin
|
|
187
185
|
return fileNames;
|
188
186
|
}
|
189
187
|
|
190
|
-
private void run(String configYaml
|
188
|
+
private void run(String configYaml)
|
191
189
|
{
|
192
190
|
ConfigSource config = getConfigFromYaml(configYaml);
|
193
191
|
runner.transaction(config, SCHEMA, 1, new Control()
|
@@ -206,16 +204,10 @@ public class TestSftpFileOutputPlugin
|
|
206
204
|
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")),
|
207
205
|
true, 2L, 3.0D, "45", Timestamp.ofEpochMilli(678L), newMap(newString("k"), newString("v")))) {
|
208
206
|
pageOutput.add(page);
|
209
|
-
if (sleep.isPresent()) {
|
210
|
-
Thread.sleep(sleep.get() * 1000);
|
211
|
-
}
|
212
207
|
}
|
213
208
|
pageOutput.commit();
|
214
209
|
committed = true;
|
215
210
|
}
|
216
|
-
catch (InterruptedException e) {
|
217
|
-
logger.debug(e.getMessage(), e);
|
218
|
-
}
|
219
211
|
finally {
|
220
212
|
if (!committed) {
|
221
213
|
pageOutput.abort();
|
@@ -374,7 +366,7 @@ public class TestSftpFileOutputPlugin
|
|
374
366
|
" default_timezone: 'UTC'";
|
375
367
|
|
376
368
|
// runner.transaction -> ...
|
377
|
-
run(configYaml
|
369
|
+
run(configYaml);
|
378
370
|
|
379
371
|
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
380
372
|
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
@@ -410,7 +402,7 @@ public class TestSftpFileOutputPlugin
|
|
410
402
|
" default_timezone: 'UTC'";
|
411
403
|
|
412
404
|
// runner.transaction -> ...
|
413
|
-
run(configYaml
|
405
|
+
run(configYaml);
|
414
406
|
|
415
407
|
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
416
408
|
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
@@ -458,7 +450,7 @@ public class TestSftpFileOutputPlugin
|
|
458
450
|
" default_timezone: 'UTC'";
|
459
451
|
|
460
452
|
// runner.transaction -> ...
|
461
|
-
run(configYaml
|
453
|
+
run(configYaml);
|
462
454
|
|
463
455
|
List<String> fileList = lsR(Lists.<String>newArrayList(), Paths.get(testFolder.getRoot().getAbsolutePath()));
|
464
456
|
assertThat(fileList, hasItem(containsString(pathPrefix + "001.00.txt")));
|
@@ -474,42 +466,6 @@ public class TestSftpFileOutputPlugin
|
|
474
466
|
}
|
475
467
|
}
|
476
468
|
|
477
|
-
@Test
|
478
|
-
public void testTimeout()
|
479
|
-
{
|
480
|
-
// setting embulk config
|
481
|
-
final String pathPrefix = "/test/testUserPassword";
|
482
|
-
String configYaml = "" +
|
483
|
-
"type: sftp\n" +
|
484
|
-
"host: " + HOST + "\n" +
|
485
|
-
"port: " + PORT + "\n" +
|
486
|
-
"user: " + USERNAME + "\n" +
|
487
|
-
"secret_key_file: " + SECRET_KEY_FILE + "\n" +
|
488
|
-
"secret_key_passphrase: " + SECRET_KEY_PASSPHRASE + "\n" +
|
489
|
-
"path_prefix: " + testFolder.getRoot().getAbsolutePath() + pathPrefix + "\n" +
|
490
|
-
"timeout: 1\n" +
|
491
|
-
"file_ext: txt\n" +
|
492
|
-
"formatter:\n" +
|
493
|
-
" type: csv\n" +
|
494
|
-
" newline: CRLF\n" +
|
495
|
-
" newline_in_field: LF\n" +
|
496
|
-
" header_line: true\n" +
|
497
|
-
" charset: UTF-8\n" +
|
498
|
-
" quote_policy: NONE\n" +
|
499
|
-
" quote: \"\\\"\"\n" +
|
500
|
-
" escape: \"\\\\\"\n" +
|
501
|
-
" null_string: \"\"\n" +
|
502
|
-
" default_timezone: 'UTC'";
|
503
|
-
|
504
|
-
// exception
|
505
|
-
exception.expect(RuntimeException.class);
|
506
|
-
exception.expectCause(CoreMatchers.<Throwable>instanceOf(FileSystemException.class));
|
507
|
-
exception.expectMessage("Could not connect to SFTP server");
|
508
|
-
|
509
|
-
// runner.transaction -> ...
|
510
|
-
run(configYaml, Optional.of(60)); // sleep 1 minute while processing
|
511
|
-
}
|
512
|
-
|
513
469
|
@Test
|
514
470
|
public void testProxyType()
|
515
471
|
{
|
metadata
CHANGED
@@ -1,43 +1,43 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-sftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Civitaspo
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-09-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
version_requirements: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ~>
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '1.0'
|
20
14
|
requirement: !ruby/object:Gem::Requirement
|
21
15
|
requirements:
|
22
16
|
- - ~>
|
23
17
|
- !ruby/object:Gem::Version
|
24
18
|
version: '1.0'
|
19
|
+
name: bundler
|
25
20
|
prerelease: false
|
26
21
|
type: :development
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rake
|
29
22
|
version_requirements: !ruby/object:Gem::Requirement
|
30
23
|
requirements:
|
31
|
-
- -
|
24
|
+
- - ~>
|
32
25
|
- !ruby/object:Gem::Version
|
33
|
-
version: '
|
26
|
+
version: '1.0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
34
28
|
requirement: !ruby/object:Gem::Requirement
|
35
29
|
requirements:
|
36
30
|
- - '>='
|
37
31
|
- !ruby/object:Gem::Version
|
38
32
|
version: '10.0'
|
33
|
+
name: rake
|
39
34
|
prerelease: false
|
40
35
|
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
41
|
description: Stores files on Sftp.
|
42
42
|
email:
|
43
43
|
- civitaspo@gmail.com
|
@@ -69,7 +69,7 @@ files:
|
|
69
69
|
- src/test/resources/id_rsa.pub
|
70
70
|
- classpath/commons-logging-1.2.jar
|
71
71
|
- classpath/commons-vfs2-2.1.1660580.2.jar
|
72
|
-
- classpath/embulk-output-sftp-0.0.
|
72
|
+
- classpath/embulk-output-sftp-0.0.8.jar
|
73
73
|
- classpath/jsch-0.1.53.jar
|
74
74
|
homepage: https://github.com/civitaspo/embulk-output-sftp
|
75
75
|
licenses:
|
Binary file
|