embulk-input-ftp 0.1.5 → 0.1.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,90 +0,0 @@
1
- @if "%DEBUG%" == "" @echo off
2
- @rem ##########################################################################
3
- @rem
4
- @rem Gradle startup script for Windows
5
- @rem
6
- @rem ##########################################################################
7
-
8
- @rem Set local scope for the variables with windows NT shell
9
- if "%OS%"=="Windows_NT" setlocal
10
-
11
- @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
12
- set DEFAULT_JVM_OPTS=
13
-
14
- set DIRNAME=%~dp0
15
- if "%DIRNAME%" == "" set DIRNAME=.
16
- set APP_BASE_NAME=%~n0
17
- set APP_HOME=%DIRNAME%
18
-
19
- @rem Find java.exe
20
- if defined JAVA_HOME goto findJavaFromJavaHome
21
-
22
- set JAVA_EXE=java.exe
23
- %JAVA_EXE% -version >NUL 2>&1
24
- if "%ERRORLEVEL%" == "0" goto init
25
-
26
- echo.
27
- echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
28
- echo.
29
- echo Please set the JAVA_HOME variable in your environment to match the
30
- echo location of your Java installation.
31
-
32
- goto fail
33
-
34
- :findJavaFromJavaHome
35
- set JAVA_HOME=%JAVA_HOME:"=%
36
- set JAVA_EXE=%JAVA_HOME%/bin/java.exe
37
-
38
- if exist "%JAVA_EXE%" goto init
39
-
40
- echo.
41
- echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
42
- echo.
43
- echo Please set the JAVA_HOME variable in your environment to match the
44
- echo location of your Java installation.
45
-
46
- goto fail
47
-
48
- :init
49
- @rem Get command-line arguments, handling Windowz variants
50
-
51
- if not "%OS%" == "Windows_NT" goto win9xME_args
52
- if "%@eval[2+2]" == "4" goto 4NT_args
53
-
54
- :win9xME_args
55
- @rem Slurp the command line arguments.
56
- set CMD_LINE_ARGS=
57
- set _SKIP=2
58
-
59
- :win9xME_args_slurp
60
- if "x%~1" == "x" goto execute
61
-
62
- set CMD_LINE_ARGS=%*
63
- goto execute
64
-
65
- :4NT_args
66
- @rem Get arguments from the 4NT Shell from JP Software
67
- set CMD_LINE_ARGS=%$
68
-
69
- :execute
70
- @rem Setup the command line
71
-
72
- set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
73
-
74
- @rem Execute Gradle
75
- "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
76
-
77
- :end
78
- @rem End local scope for the variables with windows NT shell
79
- if "%ERRORLEVEL%"=="0" goto mainEnd
80
-
81
- :fail
82
- rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
83
- rem the _cmd.exe /c_ return code!
84
- if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
85
- exit /b 1
86
-
87
- :mainEnd
88
- if "%OS%"=="Windows_NT" endlocal
89
-
90
- :omega
@@ -1,267 +0,0 @@
1
- package org.embulk.input.ftp;
2
-
3
- import java.util.concurrent.Future;
4
- import java.util.concurrent.Callable;
5
- import java.util.concurrent.CancellationException;
6
- import java.util.concurrent.ExecutionException;
7
- import java.util.concurrent.ExecutorService;
8
- import java.io.IOException;
9
- import java.io.EOFException;
10
- import java.io.InterruptedIOException;
11
- import java.nio.ByteBuffer;
12
- import java.nio.channels.ReadableByteChannel;
13
- import java.nio.channels.WritableByteChannel;
14
- import com.google.common.base.Function;
15
-
16
- public class BlockingTransfer
17
- {
18
- private final WriterChannel writerChannel;
19
- private final ReaderChannel readerChannel;
20
- private Future<?> transferCompletionFuture;
21
-
22
- public static BlockingTransfer submit(ExecutorService executor,
23
- Function<BlockingTransfer, Runnable> starterFactory)
24
- {
25
- BlockingTransfer transfer = new BlockingTransfer();
26
- final Runnable starter = starterFactory.apply(transfer);
27
- transfer.setTransferCompletionFuture(
28
- executor.submit(new Callable<Void>() {
29
- public Void call() throws Exception
30
- {
31
- starter.run();
32
- return null;
33
- }
34
- })
35
- );
36
- return transfer;
37
- }
38
-
39
- private BlockingTransfer()
40
- {
41
- this.writerChannel = new WriterChannel();
42
- this.readerChannel = new ReaderChannel();
43
- }
44
-
45
- private void setTransferCompletionFuture(Future<?> future)
46
- {
47
- this.transferCompletionFuture = future;
48
- }
49
-
50
- public ReadableByteChannel getReaderChannel()
51
- {
52
- return readerChannel;
53
- }
54
-
55
- public WritableByteChannel getWriterChannel()
56
- {
57
- return writerChannel;
58
- }
59
-
60
- public void transferFailed(Throwable exception)
61
- {
62
- readerChannel.overwriteException(exception);
63
- }
64
-
65
- void waitForTransferCompletion() throws IOException
66
- {
67
- Future<?> f = transferCompletionFuture;
68
- if (f != null) {
69
- try {
70
- f.get();
71
- } catch (CancellationException ex) {
72
- throw new InterruptedIOException();
73
- } catch (InterruptedException ex) {
74
- throw new InterruptedIOException();
75
- } catch (ExecutionException ex) {
76
- // transfer failed
77
- Throwable e = ex.getCause();
78
- if (e instanceof IOException) {
79
- throw (IOException) e;
80
- } else if (e instanceof RuntimeException) {
81
- throw (RuntimeException) e;
82
- } else if (e instanceof Error) {
83
- throw (Error) e;
84
- } else {
85
- throw new IOException(e);
86
- }
87
- }
88
- }
89
- }
90
-
91
- public class WriterChannel implements WritableByteChannel
92
- {
93
- public int write(ByteBuffer src) throws IOException
94
- {
95
- int sz = src.remaining();
96
- if (sz <= 0) {
97
- return sz;
98
- }
99
-
100
- synchronized(readerChannel) {
101
- if (!readerChannel.waitForWritable()) {
102
- return -1;
103
- }
104
-
105
- readerChannel.setBuffer(src);
106
-
107
- if (!readerChannel.waitForWritable()) { // wait for complete processing src
108
- return -1;
109
- }
110
- }
111
-
112
- return sz - src.remaining();
113
- }
114
-
115
- public boolean isOpen()
116
- {
117
- return readerChannel.isOpen();
118
- }
119
-
120
- public void close() throws IOException
121
- {
122
- readerChannel.closePeer();
123
- waitForTransferCompletion();
124
- }
125
- }
126
-
127
- private static int transferByteBuffer(ByteBuffer src, ByteBuffer dst)
128
- {
129
- int pos = dst.position();
130
-
131
- int srcrem = src.remaining();
132
- int dstrem = dst.remaining();
133
- if (dstrem < srcrem) {
134
- int lim = src.limit();
135
- try {
136
- src.limit(src.position() + dstrem);
137
- dst.put(src);
138
- } finally {
139
- src.limit(lim);
140
- }
141
- } else {
142
- dst.put(src);
143
- }
144
-
145
- return dst.position() - pos;
146
- }
147
-
148
- public class ReaderChannel implements ReadableByteChannel
149
- {
150
- private ByteBuffer buffer;
151
- private Throwable exception;
152
-
153
- public synchronized int read(ByteBuffer dst) throws IOException
154
- {
155
- if (!waitForReadable()) {
156
- return -1;
157
- }
158
-
159
- int len = transferByteBuffer(buffer, dst);
160
- if (!buffer.hasRemaining()) {
161
- setBuffer(null);
162
- notifyAll();
163
- }
164
-
165
- return len;
166
- }
167
-
168
- public synchronized boolean isOpen()
169
- {
170
- return exception == null;
171
- }
172
-
173
- public void close() throws IOException
174
- {
175
- setException(new EOFException("reader closed channel"));
176
- }
177
-
178
- private void setBuffer(ByteBuffer buffer)
179
- {
180
- this.buffer = buffer;
181
- notifyAll();
182
- }
183
-
184
- private synchronized boolean waitForWritable() throws IOException
185
- {
186
- while (buffer != null) {
187
- if (exception != null) {
188
- if (exception instanceof EOFException) {
189
- return false;
190
- }
191
- throwException();
192
- }
193
-
194
- try {
195
- wait();
196
- } catch (InterruptedException ex) {
197
- // TODO throws ClosedByInterruptException or InterruptedIOException?
198
- }
199
- }
200
-
201
- return true;
202
- }
203
-
204
- private boolean waitForReadable() throws IOException
205
- {
206
- while(buffer == null) {
207
- if (exception != null) {
208
- if (exception instanceof EOFException) {
209
- return false;
210
- }
211
- throwException();
212
- }
213
-
214
- try {
215
- wait();
216
- } catch (InterruptedException ex) {
217
- // TODO throws ClosedByInterruptException or InterruptedIOException?
218
- }
219
- }
220
-
221
- return true;
222
- }
223
-
224
- public synchronized void closePeer() throws IOException
225
- {
226
- waitForWritable();
227
- if( exception != null && !(exception instanceof EOFException)) {
228
- throwException();
229
- }
230
- setException(new EOFException("writer closed channel"));
231
- }
232
-
233
- public synchronized void setException(Throwable exception)
234
- {
235
- if (this.exception == null) {
236
- this.exception = exception;
237
- }
238
- notifyAll();
239
- }
240
-
241
- public synchronized void overwriteException(Throwable exception)
242
- {
243
- this.exception = exception;
244
- notifyAll();
245
- }
246
-
247
- public boolean hasException()
248
- {
249
- return exception != null;
250
- }
251
-
252
- public void throwException() throws IOException
253
- {
254
- Throwable ex = exception;
255
- if (ex instanceof IOException) {
256
- throw (IOException) ex;
257
- } else if (ex instanceof RuntimeException) {
258
- throw (RuntimeException) ex;
259
- } else if (ex instanceof Error) {
260
- throw (Error) ex;
261
- } else {
262
- throw new IOException(ex);
263
- }
264
- }
265
- }
266
- }
267
-
@@ -1,245 +0,0 @@
1
- package org.embulk.input.ftp;
2
-
3
- import java.util.List;
4
- import java.io.Reader;
5
- import java.io.FileReader;
6
- import java.io.StringReader;
7
- import java.io.ByteArrayInputStream;
8
- import java.io.IOException;
9
- import java.io.FileNotFoundException;
10
- import java.security.KeyStore;
11
- import java.security.NoSuchAlgorithmException;
12
- import java.security.GeneralSecurityException;
13
- import java.security.cert.CertificateFactory;
14
- import java.security.cert.X509Certificate;
15
- import java.security.cert.CertificateException;
16
- import java.security.cert.CertificateEncodingException;
17
- import java.security.KeyManagementException;
18
- import javax.net.ssl.SSLSocketFactory;
19
- import javax.net.ssl.X509TrustManager;
20
- import com.google.common.base.Optional;
21
- import com.google.common.base.Function;
22
- import com.google.common.collect.ImmutableList;
23
- import com.google.common.collect.Lists;
24
- import com.fasterxml.jackson.annotation.JsonCreator;
25
- import com.fasterxml.jackson.annotation.JsonProperty;
26
- import com.fasterxml.jackson.annotation.JsonIgnore;
27
- import org.embulk.config.Config;
28
- import org.embulk.config.ConfigDefault;
29
- import org.embulk.config.ConfigException;
30
-
31
- public class SSLPlugins
32
- {
33
- // SSLPlugins is only for SSL clients. SSL server implementation is out ouf scope.
34
-
35
- public interface SSLPluginTask
36
- {
37
- @Config("ssl_verify")
38
- @ConfigDefault("null")
39
- public Optional<Boolean> getSslVerify();
40
-
41
- @Config("ssl_verify_hostname")
42
- @ConfigDefault("true")
43
- public boolean getSslVerifyHostname();
44
-
45
- @Config("ssl_trusted_ca_cert_file")
46
- @ConfigDefault("null")
47
- public Optional<String> getSslTrustedCaCertFile();
48
-
49
- @Config("ssl_trusted_ca_cert_data")
50
- @ConfigDefault("null")
51
- public Optional<String> getSslTrustedCaCertData();
52
- }
53
-
54
- private static enum VerifyMode
55
- {
56
- NO_VERIFY,
57
- CERTIFICATES,
58
- JVM_DEFAULT;
59
- }
60
-
61
- public static class SSLPluginConfig
62
- {
63
- static SSLPluginConfig NO_VERIFY = new SSLPluginConfig(VerifyMode.NO_VERIFY, false, ImmutableList.<byte[]>of());
64
-
65
- private final VerifyMode verifyMode;
66
- private final boolean verifyHostname;
67
- private final List<X509Certificate> certificates;
68
-
69
- @JsonCreator
70
- private SSLPluginConfig(
71
- @JsonProperty("verifyMode") VerifyMode verifyMode,
72
- @JsonProperty("verifyHostname") boolean verifyHostname,
73
- @JsonProperty("certificates") List<byte[]> certificates)
74
- {
75
- this.verifyMode = verifyMode;
76
- this.verifyHostname = verifyHostname;
77
- this.certificates = ImmutableList.copyOf(
78
- Lists.transform(certificates, new Function<byte[], X509Certificate>() {
79
- public X509Certificate apply(byte[] data)
80
- {
81
- try (ByteArrayInputStream in = new ByteArrayInputStream(data)) {
82
- CertificateFactory cf = CertificateFactory.getInstance("X.509");
83
- return (X509Certificate) cf.generateCertificate(in);
84
- } catch (IOException | CertificateException ex) {
85
- throw new RuntimeException(ex);
86
- }
87
- }
88
- })
89
- );
90
- }
91
-
92
- SSLPluginConfig(List<X509Certificate> certificates, boolean verifyHostname)
93
- {
94
- this.verifyMode = VerifyMode.CERTIFICATES;
95
- this.verifyHostname = verifyHostname;
96
- this.certificates = certificates;
97
- }
98
-
99
- static SSLPluginConfig useJvmDefault(boolean verifyHostname)
100
- {
101
- return new SSLPluginConfig(VerifyMode.JVM_DEFAULT, verifyHostname, ImmutableList.<byte[]>of());
102
- }
103
-
104
- @JsonProperty("verifyMode")
105
- private VerifyMode getVerifyMode()
106
- {
107
- return verifyMode;
108
- }
109
-
110
- @JsonProperty("verifyHostname")
111
- private boolean getVerifyHostname()
112
- {
113
- return verifyHostname;
114
- }
115
-
116
- @JsonProperty("certificates")
117
- private List<byte[]> getCertData()
118
- {
119
- return Lists.transform(certificates, new Function<X509Certificate, byte[]>() {
120
- public byte[] apply(X509Certificate cert)
121
- {
122
- try {
123
- return cert.getEncoded();
124
- } catch (CertificateEncodingException ex) {
125
- throw new RuntimeException(ex);
126
- }
127
- }
128
- });
129
- }
130
-
131
- @JsonIgnore
132
- public X509TrustManager[] newTrustManager()
133
- {
134
- try {
135
- switch (verifyMode) {
136
- case NO_VERIFY:
137
- return new X509TrustManager[] { getNoVerifyTrustManager() };
138
- case CERTIFICATES:
139
- return TrustManagers.newTrustManager(certificates);
140
- default: // JVM_DEFAULT
141
- return TrustManagers.newDefaultJavaTrustManager();
142
- }
143
- } catch (IOException | GeneralSecurityException ex) {
144
- throw new RuntimeException(ex);
145
- }
146
- }
147
- }
148
-
149
- public static enum DefaultVerifyMode
150
- {
151
- VERIFY_BY_JVM_TRUSTED_CA_CERTS,
152
- NO_VERIFY;
153
- };
154
-
155
- public static SSLPluginConfig configure(SSLPluginTask task)
156
- {
157
- return configure(task, DefaultVerifyMode.VERIFY_BY_JVM_TRUSTED_CA_CERTS);
158
- }
159
-
160
- public static SSLPluginConfig configure(SSLPluginTask task, DefaultVerifyMode defaultVerifyMode)
161
- {
162
- boolean verify = task.getSslVerify().or(defaultVerifyMode != DefaultVerifyMode.NO_VERIFY);
163
- if (verify) {
164
- Optional<List<X509Certificate>> certs = readTrustedCertificates(task);
165
- if (certs.isPresent()) {
166
- return new SSLPluginConfig(certs.get(), task.getSslVerifyHostname());
167
- } else {
168
- return SSLPluginConfig.useJvmDefault(task.getSslVerifyHostname());
169
- }
170
- } else {
171
- return SSLPluginConfig.NO_VERIFY;
172
- }
173
- }
174
-
175
- private static Optional<List<X509Certificate>> readTrustedCertificates(SSLPluginTask task)
176
- {
177
- String optionName;
178
- Reader reader;
179
- if (task.getSslTrustedCaCertData().isPresent()) {
180
- optionName = "ssl_trusted_ca_cert_data";
181
- reader = new StringReader(task.getSslTrustedCaCertData().get());
182
- } else if (task.getSslTrustedCaCertFile().isPresent()) {
183
- optionName = "ssl_trusted_ca_cert_file '" + task.getSslTrustedCaCertFile().get() + "'";
184
- try {
185
- reader = new FileReader(task.getSslTrustedCaCertFile().get());
186
- } catch (IOException ex) {
187
- throw new ConfigException("Failed to open "+optionName, ex);
188
- }
189
- } else {
190
- return Optional.absent();
191
- }
192
-
193
- List<X509Certificate> certs;
194
- try (Reader r = reader) {
195
- certs = TrustManagers.readPemEncodedX509Certificates(r);
196
- if (certs.isEmpty()) {
197
- throw new ConfigException(optionName + " does not include valid X.509 PEM certificates");
198
- }
199
- } catch (CertificateException | IOException ex) {
200
- throw new ConfigException("Failed to read "+optionName, ex);
201
- }
202
-
203
- return Optional.of(certs);
204
- }
205
-
206
- public static SSLSocketFactory newSSLSocketFactory(SSLPluginConfig config, String hostname)
207
- {
208
- try {
209
- return TrustManagers.newSSLSocketFactory(
210
- null, // TODO sending client certificate is not implemented yet
211
- config.newTrustManager(),
212
- config.getVerifyHostname() ? hostname : null);
213
- } catch (KeyManagementException ex) {
214
- throw new RuntimeException(ex);
215
- }
216
- }
217
-
218
- private static class NoVerifyTrustManager
219
- implements X509TrustManager
220
- {
221
- static final NoVerifyTrustManager INSTANCE = new NoVerifyTrustManager();
222
-
223
- private NoVerifyTrustManager()
224
- { }
225
-
226
- @Override
227
- public X509Certificate[] getAcceptedIssuers()
228
- {
229
- return null;
230
- }
231
-
232
- @Override
233
- public void checkClientTrusted(X509Certificate[] certs, String authType)
234
- { }
235
-
236
- @Override
237
- public void checkServerTrusted(X509Certificate[] certs, String authType)
238
- { }
239
- }
240
-
241
- private static X509TrustManager getNoVerifyTrustManager()
242
- {
243
- return NoVerifyTrustManager.INSTANCE;
244
- }
245
- }