embulk-output-ftp 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +6 -2
- data/build.gradle +1 -1
- data/classpath/{embulk-output-ftp-0.1.4.jar → embulk-output-ftp-0.1.5.jar} +0 -0
- data/src/main/java/org/embulk/output/ftp/FtpFileOutputPlugin.java +68 -8
- data/src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java +52 -2
- metadata +3 -5
- data/src/test/java/org/embulk/output/ftp/TestSSLPlugins.java +0 -5
- data/src/test/java/org/embulk/output/ftp/TrustedManagers.java +0 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b0494ae651d3e5c17bb41c2cb08712ed7d6388d2
|
4
|
+
data.tar.gz: 9463836af3ca53940c83237a3e84126dadb8d153
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 66c737b48bc821ca444806d7d555ed17c1f7b4385f75743428a2fa3d94e72488791c0aa9883720155ac86e4d65de7e08f2c6af91afbf6ed7756cdb558116e67f
|
7
|
+
data.tar.gz: ccd8a59f0f4b58d9e02158ad757fd490c6e678768bf2e8f75aaa0a74dd0bfcfc465522b8126fe47710bf583c585218b860d410ccdd027c2e23ca4b92f8e41112
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 0.1.5 - 2016-08-08
|
2
|
+
|
3
|
+
* [maintenance] Improve connection stability [#8](https://github.com/embulk/embulk-output-ftp/pull/9)
|
4
|
+
|
1
5
|
## 0.1.4 - 2016-07-29
|
2
6
|
|
3
7
|
* [maintenance] Fix default port number generation logic [#7](https://github.com/embulk/embulk-output-ftp/pull/7)
|
data/README.md
CHANGED
@@ -141,7 +141,8 @@ FTP_TEST_SSL__PORT (default:990)
|
|
141
141
|
|
142
142
|
If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
|
143
143
|
```xml
|
144
|
-
|
144
|
+
|
145
|
+
launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE$ vi ~/Library/LaunchAgents/environment.plist
|
145
146
|
<?xml version="1.0" encoding="UTF-8"?>
|
146
147
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
147
148
|
<plist version="1.0">
|
@@ -157,7 +158,10 @@ $ vi ~/Library/LaunchAgents/environment.plist
|
|
157
158
|
launchctl setenv FTP_TEST_USER username
|
158
159
|
launchctl setenv FTP_TEST_PASSWORD password
|
159
160
|
launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE /path/to/cert.pem
|
160
|
-
launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA
|
161
|
+
launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA "-----BEGIN CERTIFICATE-----
|
162
|
+
ABCDEFG...
|
163
|
+
EFGHIJKL...
|
164
|
+
-----END CERTIFICATE-----"
|
161
165
|
</string>
|
162
166
|
</array>
|
163
167
|
<key>RunAtLoad</key>
|
data/build.gradle
CHANGED
Binary file
|
@@ -34,6 +34,8 @@ import java.io.File;
|
|
34
34
|
import java.io.FileInputStream;
|
35
35
|
import java.io.FileOutputStream;
|
36
36
|
import java.io.IOException;
|
37
|
+
import java.io.InterruptedIOException;
|
38
|
+
import java.net.ConnectException;
|
37
39
|
import java.nio.file.Path;
|
38
40
|
import java.nio.file.Paths;
|
39
41
|
import java.util.List;
|
@@ -353,14 +355,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
|
|
353
355
|
//client.setDataTimeout
|
354
356
|
//client.setAutodetectUTF8
|
355
357
|
|
356
|
-
|
357
|
-
client.connect(task.getHost(), task.getPort().get());
|
358
|
-
log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
|
359
|
-
}
|
360
|
-
else {
|
361
|
-
client.connect(task.getHost());
|
362
|
-
log.info("Connecting to {}", task.getHost());
|
363
|
-
}
|
358
|
+
client = connect(client, task);
|
364
359
|
|
365
360
|
if (task.getUser().isPresent()) {
|
366
361
|
log.info("Logging in with user {}", task.getUser().get());
|
@@ -405,6 +400,71 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
|
|
405
400
|
}
|
406
401
|
}
|
407
402
|
|
403
|
+
private static FTPClient connect(final FTPClient client, final PluginTask task) throws InterruptedIOException
|
404
|
+
{
|
405
|
+
try {
|
406
|
+
return retryExecutor()
|
407
|
+
.withRetryLimit(task.getMaxConnectionRetry())
|
408
|
+
.withInitialRetryWait(500)
|
409
|
+
.withMaxRetryWait(30 * 1000)
|
410
|
+
.runInterruptible(new Retryable<FTPClient>() {
|
411
|
+
@Override
|
412
|
+
public FTPClient call()
|
413
|
+
{
|
414
|
+
try {
|
415
|
+
if (task.getPort().isPresent()) {
|
416
|
+
client.connect(task.getHost(), task.getPort().get());
|
417
|
+
log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
|
418
|
+
}
|
419
|
+
else {
|
420
|
+
client.connect(task.getHost());
|
421
|
+
log.info("Connecting to {}", task.getHost());
|
422
|
+
}
|
423
|
+
}
|
424
|
+
catch (FTPIllegalReplyException | FTPException | IOException ex) {
|
425
|
+
throw Throwables.propagate(ex);
|
426
|
+
}
|
427
|
+
return client;
|
428
|
+
}
|
429
|
+
|
430
|
+
@Override
|
431
|
+
public boolean isRetryableException(Exception exception)
|
432
|
+
{
|
433
|
+
if (exception.getCause() != null) {
|
434
|
+
if (exception.getCause() instanceof ConnectException && exception.getMessage().contains("Connection refused")) {
|
435
|
+
return false;
|
436
|
+
}
|
437
|
+
}
|
438
|
+
return true;
|
439
|
+
}
|
440
|
+
|
441
|
+
@Override
|
442
|
+
public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
|
443
|
+
{
|
444
|
+
String message = String.format("FTP Put request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
|
445
|
+
retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
|
446
|
+
if (retryCount % 3 == 0) {
|
447
|
+
log.warn(message, exception);
|
448
|
+
}
|
449
|
+
else {
|
450
|
+
log.warn(message);
|
451
|
+
}
|
452
|
+
}
|
453
|
+
|
454
|
+
@Override
|
455
|
+
public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
|
456
|
+
{
|
457
|
+
}
|
458
|
+
});
|
459
|
+
}
|
460
|
+
catch (RetryGiveupException ex) {
|
461
|
+
throw Throwables.propagate(ex);
|
462
|
+
}
|
463
|
+
catch (InterruptedException ex) {
|
464
|
+
throw new InterruptedIOException();
|
465
|
+
}
|
466
|
+
}
|
467
|
+
|
408
468
|
static void disconnectClient(FTPClient client)
|
409
469
|
{
|
410
470
|
if (client != null && client.isConnected()) {
|
@@ -51,6 +51,7 @@ public class TestFtpFileOutputPlugin
|
|
51
51
|
private static String FTP_TEST_USER;
|
52
52
|
private static String FTP_TEST_PASSWORD;
|
53
53
|
private static String FTP_TEST_SSL_TRUSTED_CA_CERT_FILE;
|
54
|
+
private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
|
54
55
|
private static String FTP_TEST_DIRECTORY;
|
55
56
|
private static String FTP_TEST_PATH_PREFIX;
|
56
57
|
private static String LOCAL_PATH_PREFIX;
|
@@ -72,8 +73,9 @@ public class TestFtpFileOutputPlugin
|
|
72
73
|
FTP_TEST_USER = System.getenv("FTP_TEST_USER");
|
73
74
|
FTP_TEST_PASSWORD = System.getenv("FTP_TEST_PASSWORD");
|
74
75
|
FTP_TEST_SSL_TRUSTED_CA_CERT_FILE = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_FILE");
|
76
|
+
FTP_TEST_SSL_TRUSTED_CA_CERT_DATA = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_DATA");
|
75
77
|
// skip test cases, if environment variables are not set.
|
76
|
-
assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE);
|
78
|
+
assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE, FTP_TEST_SSL_TRUSTED_CA_CERT_DATA);
|
77
79
|
|
78
80
|
FTP_TEST_DIRECTORY = System.getenv("FTP_TEST_DIRECTORY") != null ? getDirectory(System.getenv("FTP_TEST_DIRECTORY")) : getDirectory("/unittest/");
|
79
81
|
FTP_TEST_PATH_PREFIX = FTP_TEST_DIRECTORY + "sample_";
|
@@ -149,6 +151,7 @@ public class TestFtpFileOutputPlugin
|
|
149
151
|
.set("password", FTP_TEST_PASSWORD)
|
150
152
|
.set("path_prefix", "my-prefix")
|
151
153
|
.set("file_ext", ".csv")
|
154
|
+
.set("max_connection_retry", 3)
|
152
155
|
.set("formatter", formatterConfig());
|
153
156
|
|
154
157
|
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
@@ -157,7 +160,7 @@ public class TestFtpFileOutputPlugin
|
|
157
160
|
}
|
158
161
|
|
159
162
|
@Test
|
160
|
-
public void
|
163
|
+
public void testTransactionWithUnverifiedSsl()
|
161
164
|
{
|
162
165
|
ConfigSource config = Exec.newConfigSource()
|
163
166
|
.set("in", inputConfig())
|
@@ -170,6 +173,29 @@ public class TestFtpFileOutputPlugin
|
|
170
173
|
.set("ssl", true)
|
171
174
|
.set("ssl_verify", false)
|
172
175
|
.set("ssl_verify_hostname", false)
|
176
|
+
.set("path_prefix", "my-prefix")
|
177
|
+
.set("file_ext", ".csv")
|
178
|
+
.set("formatter", formatterConfig());
|
179
|
+
|
180
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
181
|
+
|
182
|
+
runner.transaction(config, schema, 0, new Control());
|
183
|
+
}
|
184
|
+
|
185
|
+
@Test
|
186
|
+
public void testTransactionWithVerifiedSslWithCertFilePath()
|
187
|
+
{
|
188
|
+
ConfigSource config = Exec.newConfigSource()
|
189
|
+
.set("in", inputConfig())
|
190
|
+
.set("parser", parserConfig(schemaConfig()))
|
191
|
+
.set("type", "ftp")
|
192
|
+
.set("host", FTP_TEST_HOST)
|
193
|
+
.set("port", FTP_TEST_SSL_PORT)
|
194
|
+
.set("user", FTP_TEST_USER)
|
195
|
+
.set("password", FTP_TEST_PASSWORD)
|
196
|
+
.set("ssl", true)
|
197
|
+
.set("ssl_verify", true)
|
198
|
+
.set("ssl_verify_hostname", false)
|
173
199
|
.set("ssl_trusted_ca_cert_file", FTP_TEST_SSL_TRUSTED_CA_CERT_FILE)
|
174
200
|
.set("path_prefix", "my-prefix")
|
175
201
|
.set("file_ext", ".csv")
|
@@ -180,6 +206,30 @@ public class TestFtpFileOutputPlugin
|
|
180
206
|
runner.transaction(config, schema, 0, new Control());
|
181
207
|
}
|
182
208
|
|
209
|
+
@Test
|
210
|
+
public void testTransactionWithVerifiedSslWithCertFileData()
|
211
|
+
{
|
212
|
+
ConfigSource config = Exec.newConfigSource()
|
213
|
+
.set("in", inputConfig())
|
214
|
+
.set("parser", parserConfig(schemaConfig()))
|
215
|
+
.set("type", "ftp")
|
216
|
+
.set("host", FTP_TEST_HOST)
|
217
|
+
.set("port", FTP_TEST_SSL_PORT)
|
218
|
+
.set("user", FTP_TEST_USER)
|
219
|
+
.set("password", FTP_TEST_PASSWORD)
|
220
|
+
.set("ssl", true)
|
221
|
+
.set("ssl_verify", true)
|
222
|
+
.set("ssl_verify_hostname", false)
|
223
|
+
.set("ssl_trusted_ca_cert_data", FTP_TEST_SSL_TRUSTED_CA_CERT_DATA)
|
224
|
+
.set("path_prefix", "my-prefix")
|
225
|
+
.set("file_ext", ".csv")
|
226
|
+
.set("formatter", formatterConfig());
|
227
|
+
|
228
|
+
Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
|
229
|
+
|
230
|
+
runner.transaction(config, schema, 0, new Control());
|
231
|
+
}
|
232
|
+
|
183
233
|
@Test
|
184
234
|
public void testResume()
|
185
235
|
{
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: embulk-output-ftp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Satoshi Akama
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-08-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -63,13 +63,11 @@ files:
|
|
63
63
|
- src/main/java/org/embulk/output/ftp/SSLPlugins.java
|
64
64
|
- src/main/java/org/embulk/output/ftp/TrustManagers.java
|
65
65
|
- src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java
|
66
|
-
- src/test/java/org/embulk/output/ftp/TestSSLPlugins.java
|
67
|
-
- src/test/java/org/embulk/output/ftp/TrustedManagers.java
|
68
66
|
- src/test/resources/sample_01.csv
|
69
67
|
- src/test/resources/sample_02.csv
|
70
68
|
- classpath/bcpkix-jdk15on-1.52.jar
|
71
69
|
- classpath/bcprov-jdk15on-1.52.jar
|
72
|
-
- classpath/embulk-output-ftp-0.1.
|
70
|
+
- classpath/embulk-output-ftp-0.1.5.jar
|
73
71
|
- classpath/ftp4j-1.7.2.jar
|
74
72
|
homepage: https://github.com/embulk/embulk-output-ftp
|
75
73
|
licenses:
|