embulk-output-ftp 0.1.4 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 99f92ff252075824f860a2929aa8dc90609b3760
4
- data.tar.gz: ae7284ac04e110cb17b02f5830a2bdc52c5e309c
3
+ metadata.gz: b0494ae651d3e5c17bb41c2cb08712ed7d6388d2
4
+ data.tar.gz: 9463836af3ca53940c83237a3e84126dadb8d153
5
5
  SHA512:
6
- metadata.gz: 6f867e7103b201e4ddbab093001d6fa657d4932c469bae0baca717ce618b45d6e72521ffae58f0aab43c93d171250e366cf3666b204be31b7906fdbd00505c92
7
- data.tar.gz: 36e2127321a3cef31515fe3b037c35db61c3187afdf5fd7010e8de2e64c9eca48f6eb1eee83f55498f79f86b7ed232d4c1e54c9e06ef487ecdd5a64e9b7d7382
6
+ metadata.gz: 66c737b48bc821ca444806d7d555ed17c1f7b4385f75743428a2fa3d94e72488791c0aa9883720155ac86e4d65de7e08f2c6af91afbf6ed7756cdb558116e67f
7
+ data.tar.gz: ccd8a59f0f4b58d9e02158ad757fd490c6e678768bf2e8f75aaa0a74dd0bfcfc465522b8126fe47710bf583c585218b860d410ccdd027c2e23ca4b92f8e41112
data/CHANGELOG.md CHANGED
@@ -1,3 +1,7 @@
1
+ ## 0.1.5 - 2016-08-08
2
+
3
+ * [maintenance] Improve connection stability [#8](https://github.com/embulk/embulk-output-ftp/pull/9)
4
+
1
5
  ## 0.1.4 - 2016-07-29
2
6
 
3
7
  * [maintenance] Fix default port number generation logic [#7](https://github.com/embulk/embulk-output-ftp/pull/7)
data/README.md CHANGED
@@ -141,7 +141,8 @@ FTP_TEST_SSL__PORT (default:990)
141
141
 
142
142
  If you're using Mac OS X El Capitan and GUI Applications(IDE), like as follows.
143
143
  ```xml
144
- $ vi ~/Library/LaunchAgents/environment.plist
144
+
145
+ launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE$ vi ~/Library/LaunchAgents/environment.plist
145
146
  <?xml version="1.0" encoding="UTF-8"?>
146
147
  <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
147
148
  <plist version="1.0">
@@ -157,7 +158,10 @@ $ vi ~/Library/LaunchAgents/environment.plist
157
158
  launchctl setenv FTP_TEST_USER username
158
159
  launchctl setenv FTP_TEST_PASSWORD password
159
160
  launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_FILE /path/to/cert.pem
160
- launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA /path/to/cert.pem
161
+ launchctl setenv FTP_TEST_SSL_TRUSTED_CA_CERT_DATA "-----BEGIN CERTIFICATE-----
162
+ ABCDEFG...
163
+ EFGHIJKL...
164
+ -----END CERTIFICATE-----"
161
165
  </string>
162
166
  </array>
163
167
  <key>RunAtLoad</key>
data/build.gradle CHANGED
@@ -14,7 +14,7 @@ configurations {
14
14
  provided
15
15
  }
16
16
 
17
- version = "0.1.4"
17
+ version = "0.1.5"
18
18
 
19
19
  sourceCompatibility = 1.7
20
20
  targetCompatibility = 1.7
@@ -34,6 +34,8 @@ import java.io.File;
34
34
  import java.io.FileInputStream;
35
35
  import java.io.FileOutputStream;
36
36
  import java.io.IOException;
37
+ import java.io.InterruptedIOException;
38
+ import java.net.ConnectException;
37
39
  import java.nio.file.Path;
38
40
  import java.nio.file.Paths;
39
41
  import java.util.List;
@@ -353,14 +355,7 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
353
355
  //client.setDataTimeout
354
356
  //client.setAutodetectUTF8
355
357
 
356
- if (task.getPort().isPresent()) {
357
- client.connect(task.getHost(), task.getPort().get());
358
- log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
359
- }
360
- else {
361
- client.connect(task.getHost());
362
- log.info("Connecting to {}", task.getHost());
363
- }
358
+ client = connect(client, task);
364
359
 
365
360
  if (task.getUser().isPresent()) {
366
361
  log.info("Logging in with user {}", task.getUser().get());
@@ -405,6 +400,71 @@ public class FtpFileOutputPlugin implements FileOutputPlugin
405
400
  }
406
401
  }
407
402
 
403
+ private static FTPClient connect(final FTPClient client, final PluginTask task) throws InterruptedIOException
404
+ {
405
+ try {
406
+ return retryExecutor()
407
+ .withRetryLimit(task.getMaxConnectionRetry())
408
+ .withInitialRetryWait(500)
409
+ .withMaxRetryWait(30 * 1000)
410
+ .runInterruptible(new Retryable<FTPClient>() {
411
+ @Override
412
+ public FTPClient call()
413
+ {
414
+ try {
415
+ if (task.getPort().isPresent()) {
416
+ client.connect(task.getHost(), task.getPort().get());
417
+ log.info("Connecting to {}:{}", task.getHost(), task.getPort().get());
418
+ }
419
+ else {
420
+ client.connect(task.getHost());
421
+ log.info("Connecting to {}", task.getHost());
422
+ }
423
+ }
424
+ catch (FTPIllegalReplyException | FTPException | IOException ex) {
425
+ throw Throwables.propagate(ex);
426
+ }
427
+ return client;
428
+ }
429
+
430
+ @Override
431
+ public boolean isRetryableException(Exception exception)
432
+ {
433
+ if (exception.getCause() != null) {
434
+ if (exception.getCause() instanceof ConnectException && exception.getMessage().contains("Connection refused")) {
435
+ return false;
436
+ }
437
+ }
438
+ return true;
439
+ }
440
+
441
+ @Override
442
+ public void onRetry(Exception exception, int retryCount, int retryLimit, int retryWait) throws RetryGiveupException
443
+ {
444
+ String message = String.format("FTP Put request failed. Retrying %d/%d after %d seconds. Message: %s: %s",
445
+ retryCount, retryLimit, retryWait / 1000, exception.getClass(), exception.getMessage());
446
+ if (retryCount % 3 == 0) {
447
+ log.warn(message, exception);
448
+ }
449
+ else {
450
+ log.warn(message);
451
+ }
452
+ }
453
+
454
+ @Override
455
+ public void onGiveup(Exception firstException, Exception lastException) throws RetryGiveupException
456
+ {
457
+ }
458
+ });
459
+ }
460
+ catch (RetryGiveupException ex) {
461
+ throw Throwables.propagate(ex);
462
+ }
463
+ catch (InterruptedException ex) {
464
+ throw new InterruptedIOException();
465
+ }
466
+ }
467
+
408
468
  static void disconnectClient(FTPClient client)
409
469
  {
410
470
  if (client != null && client.isConnected()) {
@@ -51,6 +51,7 @@ public class TestFtpFileOutputPlugin
51
51
  private static String FTP_TEST_USER;
52
52
  private static String FTP_TEST_PASSWORD;
53
53
  private static String FTP_TEST_SSL_TRUSTED_CA_CERT_FILE;
54
+ private static String FTP_TEST_SSL_TRUSTED_CA_CERT_DATA;
54
55
  private static String FTP_TEST_DIRECTORY;
55
56
  private static String FTP_TEST_PATH_PREFIX;
56
57
  private static String LOCAL_PATH_PREFIX;
@@ -72,8 +73,9 @@ public class TestFtpFileOutputPlugin
72
73
  FTP_TEST_USER = System.getenv("FTP_TEST_USER");
73
74
  FTP_TEST_PASSWORD = System.getenv("FTP_TEST_PASSWORD");
74
75
  FTP_TEST_SSL_TRUSTED_CA_CERT_FILE = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_FILE");
76
+ FTP_TEST_SSL_TRUSTED_CA_CERT_DATA = System.getenv("FTP_TEST_SSL_TRUSTED_CA_CERT_DATA");
75
77
  // skip test cases, if environment variables are not set.
76
- assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE);
78
+ assumeNotNull(FTP_TEST_HOST, FTP_TEST_USER, FTP_TEST_PASSWORD, FTP_TEST_SSL_TRUSTED_CA_CERT_FILE, FTP_TEST_SSL_TRUSTED_CA_CERT_DATA);
77
79
 
78
80
  FTP_TEST_DIRECTORY = System.getenv("FTP_TEST_DIRECTORY") != null ? getDirectory(System.getenv("FTP_TEST_DIRECTORY")) : getDirectory("/unittest/");
79
81
  FTP_TEST_PATH_PREFIX = FTP_TEST_DIRECTORY + "sample_";
@@ -149,6 +151,7 @@ public class TestFtpFileOutputPlugin
149
151
  .set("password", FTP_TEST_PASSWORD)
150
152
  .set("path_prefix", "my-prefix")
151
153
  .set("file_ext", ".csv")
154
+ .set("max_connection_retry", 3)
152
155
  .set("formatter", formatterConfig());
153
156
 
154
157
  Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
@@ -157,7 +160,7 @@ public class TestFtpFileOutputPlugin
157
160
  }
158
161
 
159
162
  @Test
160
- public void testTransactionWithSsl()
163
+ public void testTransactionWithUnverifiedSsl()
161
164
  {
162
165
  ConfigSource config = Exec.newConfigSource()
163
166
  .set("in", inputConfig())
@@ -170,6 +173,29 @@ public class TestFtpFileOutputPlugin
170
173
  .set("ssl", true)
171
174
  .set("ssl_verify", false)
172
175
  .set("ssl_verify_hostname", false)
176
+ .set("path_prefix", "my-prefix")
177
+ .set("file_ext", ".csv")
178
+ .set("formatter", formatterConfig());
179
+
180
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
181
+
182
+ runner.transaction(config, schema, 0, new Control());
183
+ }
184
+
185
+ @Test
186
+ public void testTransactionWithVerifiedSslWithCertFilePath()
187
+ {
188
+ ConfigSource config = Exec.newConfigSource()
189
+ .set("in", inputConfig())
190
+ .set("parser", parserConfig(schemaConfig()))
191
+ .set("type", "ftp")
192
+ .set("host", FTP_TEST_HOST)
193
+ .set("port", FTP_TEST_SSL_PORT)
194
+ .set("user", FTP_TEST_USER)
195
+ .set("password", FTP_TEST_PASSWORD)
196
+ .set("ssl", true)
197
+ .set("ssl_verify", true)
198
+ .set("ssl_verify_hostname", false)
173
199
  .set("ssl_trusted_ca_cert_file", FTP_TEST_SSL_TRUSTED_CA_CERT_FILE)
174
200
  .set("path_prefix", "my-prefix")
175
201
  .set("file_ext", ".csv")
@@ -180,6 +206,30 @@ public class TestFtpFileOutputPlugin
180
206
  runner.transaction(config, schema, 0, new Control());
181
207
  }
182
208
 
209
+ @Test
210
+ public void testTransactionWithVerifiedSslWithCertFileData()
211
+ {
212
+ ConfigSource config = Exec.newConfigSource()
213
+ .set("in", inputConfig())
214
+ .set("parser", parserConfig(schemaConfig()))
215
+ .set("type", "ftp")
216
+ .set("host", FTP_TEST_HOST)
217
+ .set("port", FTP_TEST_SSL_PORT)
218
+ .set("user", FTP_TEST_USER)
219
+ .set("password", FTP_TEST_PASSWORD)
220
+ .set("ssl", true)
221
+ .set("ssl_verify", true)
222
+ .set("ssl_verify_hostname", false)
223
+ .set("ssl_trusted_ca_cert_data", FTP_TEST_SSL_TRUSTED_CA_CERT_DATA)
224
+ .set("path_prefix", "my-prefix")
225
+ .set("file_ext", ".csv")
226
+ .set("formatter", formatterConfig());
227
+
228
+ Schema schema = config.getNested("parser").loadConfig(CsvParserPlugin.PluginTask.class).getSchemaConfig().toSchema();
229
+
230
+ runner.transaction(config, schema, 0, new Control());
231
+ }
232
+
183
233
  @Test
184
234
  public void testResume()
185
235
  {
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: embulk-output-ftp
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.5
5
5
  platform: ruby
6
6
  authors:
7
7
  - Satoshi Akama
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-07-29 00:00:00.000000000 Z
11
+ date: 2016-08-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
@@ -63,13 +63,11 @@ files:
63
63
  - src/main/java/org/embulk/output/ftp/SSLPlugins.java
64
64
  - src/main/java/org/embulk/output/ftp/TrustManagers.java
65
65
  - src/test/java/org/embulk/output/ftp/TestFtpFileOutputPlugin.java
66
- - src/test/java/org/embulk/output/ftp/TestSSLPlugins.java
67
- - src/test/java/org/embulk/output/ftp/TrustedManagers.java
68
66
  - src/test/resources/sample_01.csv
69
67
  - src/test/resources/sample_02.csv
70
68
  - classpath/bcpkix-jdk15on-1.52.jar
71
69
  - classpath/bcprov-jdk15on-1.52.jar
72
- - classpath/embulk-output-ftp-0.1.4.jar
70
+ - classpath/embulk-output-ftp-0.1.5.jar
73
71
  - classpath/ftp4j-1.7.2.jar
74
72
  homepage: https://github.com/embulk/embulk-output-ftp
75
73
  licenses:
@@ -1,5 +0,0 @@
1
- package org.embulk.output.ftp;
2
-
3
- public class TestSSLPlugins
4
- {
5
- }
@@ -1,5 +0,0 @@
1
- package org.embulk.output.ftp;
2
-
3
- public class TrustedManagers
4
- {
5
- }