iostreams 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +5 -22
- data/Rakefile +45 -0
- data/docs/CLAUDE.md +9 -0
- data/docs/config.md +157 -0
- data/docs/copy_files.md +75 -0
- data/docs/extensions.md +111 -0
- data/docs/formats.md +188 -0
- data/docs/index.md +388 -0
- data/docs/path.md +652 -0
- data/docs/pgp.md +436 -0
- data/docs/streams.md +337 -0
- data/docs/tutorial.md +483 -0
- data/docs/upgrading.md +217 -0
- data/lib/io_streams/builder.rb +62 -2
- data/lib/io_streams/bzip2/reader.rb +25 -2
- data/lib/io_streams/bzip2/writer.rb +26 -2
- data/lib/io_streams/encode/reader.rb +4 -0
- data/lib/io_streams/encode/writer.rb +4 -0
- data/lib/io_streams/errors.rb +4 -0
- data/lib/io_streams/gzip/reader.rb +4 -0
- data/lib/io_streams/gzip/writer.rb +11 -2
- data/lib/io_streams/io_streams.rb +111 -1
- data/lib/io_streams/line/reader.rb +7 -2
- data/lib/io_streams/path.rb +115 -6
- data/lib/io_streams/paths/file.rb +47 -1
- data/lib/io_streams/paths/http.rb +45 -4
- data/lib/io_streams/paths/s3.rb +66 -15
- data/lib/io_streams/paths/sftp/net_ssh.rb +104 -0
- data/lib/io_streams/paths/sftp.rb +97 -57
- data/lib/io_streams/pgp/reader.rb +42 -2
- data/lib/io_streams/pgp/writer.rb +26 -6
- data/lib/io_streams/pgp.rb +78 -21
- data/lib/io_streams/reader.rb +10 -1
- data/lib/io_streams/record/reader.rb +72 -2
- data/lib/io_streams/stream.rb +12 -7
- data/lib/io_streams/symmetric_encryption/reader.rb +4 -0
- data/lib/io_streams/symmetric_encryption/writer.rb +4 -0
- data/lib/io_streams/tabular/header.rb +31 -4
- data/lib/io_streams/tabular/parser/base.rb +10 -0
- data/lib/io_streams/tabular/parser/csv.rb +5 -0
- data/lib/io_streams/tabular/parser/fixed.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +6 -2
- data/lib/io_streams/utils.rb +31 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +10 -1
- data/lib/io_streams/xlsx/reader.rb +5 -1
- data/lib/io_streams/zip/reader.rb +4 -0
- data/lib/io_streams/zip/writer.rb +4 -0
- metadata +24 -7
|
@@ -18,6 +18,11 @@ module IOStreams
|
|
|
18
18
|
# Full url showing all the optional elements that can be set via the url:
|
|
19
19
|
# https://username:password@hostname/path/file_name
|
|
20
20
|
#
|
|
21
|
+
# SECURITY WARNING:
|
|
22
|
+
# A username and password supplied in the url remain part of it, so `#to_s` and `#url`
|
|
23
|
+
# return them, as does any log or error message that includes the path.
|
|
24
|
+
# Supply them with the `username:` and `password:` arguments instead.
|
|
25
|
+
#
|
|
21
26
|
# username: [String]
|
|
22
27
|
# When supplied, basic authentication is used with the username and password.
|
|
23
28
|
#
|
|
@@ -108,7 +113,9 @@ module IOStreams
|
|
|
108
113
|
request.basic_auth(username, password) if username && same_origin?(uri)
|
|
109
114
|
|
|
110
115
|
http.request(request) do |response|
|
|
111
|
-
|
|
116
|
+
if response.is_a?(Net::HTTPNotFound)
|
|
117
|
+
raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{without_credentials(uri)}")
|
|
118
|
+
end
|
|
112
119
|
if response.is_a?(Net::HTTPUnauthorized)
|
|
113
120
|
raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
|
|
114
121
|
end
|
|
@@ -117,7 +124,10 @@ module IOStreams
|
|
|
117
124
|
raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
|
|
118
125
|
|
|
119
126
|
location = response["location"]
|
|
120
|
-
|
|
127
|
+
unless location
|
|
128
|
+
raise(IOStreams::Errors::CommunicationsFailure,
|
|
129
|
+
"Redirect missing location header: #{without_credentials(uri)}")
|
|
130
|
+
end
|
|
121
131
|
|
|
122
132
|
# Resolve relative redirects against the current uri.
|
|
123
133
|
new_uri = uri.merge(location)
|
|
@@ -129,7 +139,7 @@ module IOStreams
|
|
|
129
139
|
end
|
|
130
140
|
|
|
131
141
|
# Since Net::HTTP download only supports a push stream, write it to a tempfile first.
|
|
132
|
-
Utils.
|
|
142
|
+
Utils.private_temp_file("iostreams_http") do |file_name|
|
|
133
143
|
download_to_file(response, file_name)
|
|
134
144
|
# Return a read stream
|
|
135
145
|
result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
|
@@ -141,15 +151,46 @@ module IOStreams
|
|
|
141
151
|
|
|
142
152
|
# Validate that the host may be contacted, and that the scheme is still http(s)
|
|
143
153
|
# after following a redirect.
|
|
154
|
+
#
|
|
155
|
+
# A redirect must also be within the allowed paths, see `IOStreams.add_allowed_path`.
|
|
144
156
|
def validate_uri!(uri)
|
|
145
157
|
unless %w[http https].include?(uri.scheme)
|
|
146
|
-
raise(IOStreams::Errors::CommunicationsFailure,
|
|
158
|
+
raise(IOStreams::Errors::CommunicationsFailure,
|
|
159
|
+
"Invalid redirect, only http and https are supported: #{without_credentials(uri)}")
|
|
147
160
|
end
|
|
161
|
+
authorize_location!(http_location(uri)) unless IOStreams.allowed_paths.empty?
|
|
148
162
|
return if allow_hosts.nil? || allow_hosts.include?(uri.hostname)
|
|
149
163
|
|
|
150
164
|
raise(IOStreams::Errors::CommunicationsFailure, "Host not in the allowed list of hosts: #{uri.hostname}")
|
|
151
165
|
end
|
|
152
166
|
|
|
167
|
+
# Returns [String] the scheme, host, port and path of the url, which is compared against the allowed paths.
|
|
168
|
+
def allowed_location
|
|
169
|
+
http_location(original_uri)
|
|
170
|
+
end
|
|
171
|
+
|
|
172
|
+
# Returns [String] the scheme, host, port and path of the supplied uri, without any credentials or query.
|
|
173
|
+
#
|
|
174
|
+
# The path is decoded and `.` and `..` resolved, the way most web servers resolve them, so that for
|
|
175
|
+
# example `%2e%2e` cannot be used to leave an allowed path. A backslash is treated as a `/`.
|
|
176
|
+
def http_location(uri)
|
|
177
|
+
raise(Errors::AccessDenied, "Access denied: #{without_credentials(uri)} has no host") if uri.host.to_s.empty?
|
|
178
|
+
|
|
179
|
+
path = URI.decode_uri_component(uri.path).tr("\\", "/")
|
|
180
|
+
"#{uri.scheme}://#{uri.host.downcase}:#{uri.port}#{normalize_path(path)}".chomp("/")
|
|
181
|
+
rescue ArgumentError => e
|
|
182
|
+
raise(Errors::AccessDenied, "Access denied to #{without_credentials(uri)}: #{e.message}")
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
# Returns [String] the uri without any user name or password, for use in error messages.
|
|
186
|
+
def without_credentials(uri)
|
|
187
|
+
return uri.to_s unless uri.user
|
|
188
|
+
|
|
189
|
+
uri = uri.dup
|
|
190
|
+
uri.user = nil
|
|
191
|
+
uri.to_s
|
|
192
|
+
end
|
|
193
|
+
|
|
153
194
|
def same_origin?(uri)
|
|
154
195
|
original = original_uri
|
|
155
196
|
uri.scheme == original.scheme && uri.hostname == original.hostname && uri.port == original.port
|
data/lib/io_streams/paths/s3.rb
CHANGED
|
@@ -21,6 +21,15 @@ module IOStreams
|
|
|
21
21
|
# s3://my-bucket-name/file_name.txt
|
|
22
22
|
# s3://my-bucket-name/some_path/file_name.csv
|
|
23
23
|
#
|
|
24
|
+
# Any query string in the url is added to the S3 request parameters, for example:
|
|
25
|
+
# s3://my-bucket-name/file_name.csv?acl=bucket-owner-full-control
|
|
26
|
+
#
|
|
27
|
+
# SECURITY WARNING:
|
|
28
|
+
# Do not interpolate untrusted file names into the url, since a name such as
|
|
29
|
+
# `file.csv?acl=public-read` would set request parameters.
|
|
30
|
+
# Instead join untrusted names onto the path, which does not parse them as a query:
|
|
31
|
+
# IOStreams.path("s3://my-bucket-name/uploads").join(untrusted_name)
|
|
32
|
+
#
|
|
24
33
|
# access_key_id: [String]
|
|
25
34
|
# AWS Access Key Id to use to access this bucket.
|
|
26
35
|
#
|
|
@@ -185,6 +194,7 @@ module IOStreams
|
|
|
185
194
|
end
|
|
186
195
|
|
|
187
196
|
def delete
|
|
197
|
+
authorize!
|
|
188
198
|
client.delete_object(bucket: bucket_name, key: path)
|
|
189
199
|
self
|
|
190
200
|
rescue Aws::S3::Errors::NotFound
|
|
@@ -192,6 +202,7 @@ module IOStreams
|
|
|
192
202
|
end
|
|
193
203
|
|
|
194
204
|
def exist?
|
|
205
|
+
authorize!
|
|
195
206
|
client.head_object(bucket: bucket_name, key: path)
|
|
196
207
|
true
|
|
197
208
|
rescue Aws::S3::Errors::NotFound
|
|
@@ -216,6 +227,8 @@ module IOStreams
|
|
|
216
227
|
target = IOStreams.new(target_path)
|
|
217
228
|
return super(target, convert: convert, **args) unless target.is_a?(self.class)
|
|
218
229
|
|
|
230
|
+
authorize!
|
|
231
|
+
target.authorize!
|
|
219
232
|
source_name = ::File.join(bucket_name, path)
|
|
220
233
|
client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
|
|
221
234
|
target
|
|
@@ -230,6 +243,8 @@ module IOStreams
|
|
|
230
243
|
return super(source, convert: convert, **args)
|
|
231
244
|
end
|
|
232
245
|
|
|
246
|
+
authorize!
|
|
247
|
+
source.authorize!
|
|
233
248
|
source_name = ::File.join(source.bucket_name, source.path)
|
|
234
249
|
client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
|
|
235
250
|
end
|
|
@@ -244,6 +259,7 @@ module IOStreams
|
|
|
244
259
|
end
|
|
245
260
|
|
|
246
261
|
def size
|
|
262
|
+
authorize!
|
|
247
263
|
client.head_object(bucket: bucket_name, key: path).content_length
|
|
248
264
|
rescue Aws::S3::Errors::NotFound
|
|
249
265
|
nil
|
|
@@ -254,7 +270,7 @@ module IOStreams
|
|
|
254
270
|
# Read from AWS S3 file.
|
|
255
271
|
def stream_reader(&block)
|
|
256
272
|
# Since S3 download only supports a push stream, write it to a tempfile first.
|
|
257
|
-
Utils.
|
|
273
|
+
Utils.private_temp_file("iostreams_s3") do |file_name|
|
|
258
274
|
read_file(file_name)
|
|
259
275
|
|
|
260
276
|
::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
|
@@ -263,6 +279,7 @@ module IOStreams
|
|
|
263
279
|
|
|
264
280
|
# Shortcut method if caller has a filename already with no other streams applied:
|
|
265
281
|
def read_file(file_name)
|
|
282
|
+
authorize!
|
|
266
283
|
::File.open(file_name, "wb") do |file|
|
|
267
284
|
client.get_object(options.merge(response_target: file, bucket: bucket_name, key: path))
|
|
268
285
|
end
|
|
@@ -277,7 +294,7 @@ module IOStreams
|
|
|
277
294
|
# aborted.
|
|
278
295
|
def stream_writer(&block)
|
|
279
296
|
# Since S3 upload only supports a pull stream, write it to a tempfile first.
|
|
280
|
-
Utils.
|
|
297
|
+
Utils.private_temp_file("iostreams_s3") do |file_name|
|
|
281
298
|
result = ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
|
|
282
299
|
|
|
283
300
|
# Upload file only once all data has been written to it
|
|
@@ -288,6 +305,7 @@ module IOStreams
|
|
|
288
305
|
|
|
289
306
|
# Shortcut method if caller has a filename already with no other streams applied:
|
|
290
307
|
def write_file(file_name)
|
|
308
|
+
authorize!
|
|
291
309
|
if ::File.size(file_name) > MULTIPART_UPLOAD_SIZE
|
|
292
310
|
# Use multipart file upload
|
|
293
311
|
s3 = Aws::S3::Resource.new(client: client)
|
|
@@ -308,29 +326,26 @@ module IOStreams
|
|
|
308
326
|
case_sensitive: case_sensitive, directories: directories, hidden: hidden)
|
|
309
327
|
end
|
|
310
328
|
|
|
329
|
+
authorize!
|
|
311
330
|
matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
|
|
312
331
|
|
|
313
332
|
# When the pattern includes an exact file name without any pattern characters
|
|
314
333
|
if matcher.pattern.nil?
|
|
315
|
-
yield(matcher.path) if matcher.path.exist?
|
|
334
|
+
yield(matcher.path) if allowed_child?(matcher.path) && matcher.path.exist?
|
|
316
335
|
return
|
|
317
336
|
end
|
|
318
337
|
|
|
319
338
|
prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
# Fetches upto 1,000 entries at a time
|
|
323
|
-
resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
|
|
324
|
-
resp.contents.each do |object|
|
|
325
|
-
next if !directories && object.key.end_with?("/")
|
|
339
|
+
each_object(prefix) do |name, object|
|
|
340
|
+
next if !directories && object.key.end_with?("/")
|
|
326
341
|
|
|
327
|
-
|
|
328
|
-
|
|
342
|
+
file_name = ::File.join("s3://", name, object.key)
|
|
343
|
+
next unless matcher.match?(file_name)
|
|
329
344
|
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
345
|
+
child = child_path(name, object.key)
|
|
346
|
+
next unless allowed_child?(child)
|
|
347
|
+
|
|
348
|
+
yield(child, object.to_h)
|
|
334
349
|
end
|
|
335
350
|
nil
|
|
336
351
|
end
|
|
@@ -344,6 +359,42 @@ module IOStreams
|
|
|
344
359
|
def client
|
|
345
360
|
@client ||= ::Aws::S3::Client.new(@client_options)
|
|
346
361
|
end
|
|
362
|
+
|
|
363
|
+
private
|
|
364
|
+
|
|
365
|
+
# Yields the bucket name and each object in the bucket whose key starts with the supplied prefix.
|
|
366
|
+
def each_object(prefix)
|
|
367
|
+
token = nil
|
|
368
|
+
loop do
|
|
369
|
+
# Fetches upto 1,000 entries at a time
|
|
370
|
+
resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
|
|
371
|
+
resp.contents.each { |object| yield(resp.name, object) }
|
|
372
|
+
token = resp.next_continuation_token
|
|
373
|
+
break if token.nil?
|
|
374
|
+
end
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
# Returns [String] the bucket and key, which is compared against the allowed paths.
|
|
378
|
+
#
|
|
379
|
+
# S3 treats `.` and `..` in a key as ordinary characters, but other services that implement the
|
|
380
|
+
# S3 API may resolve them, so keys containing them are denied.
|
|
381
|
+
def allowed_location
|
|
382
|
+
if path.split("/").intersect?([".", ".."])
|
|
383
|
+
raise(Errors::AccessDenied, "Access denied to #{self}: '.' and '..' are not allowed in S3 keys")
|
|
384
|
+
end
|
|
385
|
+
|
|
386
|
+
to_s.sub(%r{/+\z}, "")
|
|
387
|
+
end
|
|
388
|
+
|
|
389
|
+
# Set the key directly rather than parsing it as part of a URL, since a key can contain
|
|
390
|
+
# characters such as `?`, `+` or `%` that a URL parser would treat as a query or as escapes.
|
|
391
|
+
#
|
|
392
|
+
# The child uses this path's client, so that it has the same credentials and region.
|
|
393
|
+
def child_path(bucket_name, key)
|
|
394
|
+
child = self.class.new("s3://#{bucket_name}", client: client)
|
|
395
|
+
child.path = key.dup.freeze
|
|
396
|
+
child
|
|
397
|
+
end
|
|
347
398
|
end
|
|
348
399
|
end
|
|
349
400
|
end
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
module IOStreams
|
|
2
|
+
module Paths
|
|
3
|
+
class SFTP < IOStreams::Path
|
|
4
|
+
# Translates the ssh options supplied for the sftp executable into options for net-ssh,
|
|
5
|
+
# which `SFTP#each_child` uses to list files instead of the sftp executable.
|
|
6
|
+
#
|
|
7
|
+
# The net-ssh options match those that SFTP passes to the sftp executable.
|
|
8
|
+
module NetSSH
|
|
9
|
+
# The ssh options that net-ssh supports.
|
|
10
|
+
OPTIONS = %w[
|
|
11
|
+
HostKey IdentityKey IdentityFile UserKnownHostsFile StrictHostKeyChecking
|
|
12
|
+
ConnectTimeout ServerAliveInterval ServerAliveCountMax LogLevel
|
|
13
|
+
].freeze
|
|
14
|
+
|
|
15
|
+
# Yields [Hash] the net-ssh options.
|
|
16
|
+
#
|
|
17
|
+
# Raises [ArgumentError] when `ssh_options` includes an option that is not in `OPTIONS`.
|
|
18
|
+
def self.options(ssh_options, port:, password:)
|
|
19
|
+
options = {port: port, max_pkt_size: 65_536, non_interactive: true}
|
|
20
|
+
options[:logger] = IOStreams.logger if IOStreams.logger
|
|
21
|
+
# Like the sftp executable, only use the password when one is supplied, and otherwise only public keys.
|
|
22
|
+
if password
|
|
23
|
+
options[:password] = password
|
|
24
|
+
options[:auth_methods] = %w[password keyboard-interactive]
|
|
25
|
+
else
|
|
26
|
+
options[:auth_methods] = %w[publickey]
|
|
27
|
+
end
|
|
28
|
+
# Like the sftp executable, which uses `StrictHostKeyChecking=yes`, instead of the
|
|
29
|
+
# net-ssh default of trusting a host key the first time it is seen.
|
|
30
|
+
options[:verify_host_key] = :always
|
|
31
|
+
|
|
32
|
+
ssh_options.each_pair { |key, value| add(options, key, value) }
|
|
33
|
+
return yield(options) unless ssh_options.key?("HostKey")
|
|
34
|
+
|
|
35
|
+
# Like the sftp executable, the host key replaces the user's known_hosts file.
|
|
36
|
+
Utils.private_temp_file("iostreams-sftp-known-hosts") do |file_name|
|
|
37
|
+
::File.binwrite(file_name, ssh_options["HostKey"])
|
|
38
|
+
options[:user_known_hosts_file] = [file_name]
|
|
39
|
+
yield(options)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def self.add(options, key, value)
|
|
44
|
+
case key
|
|
45
|
+
when "HostKey"
|
|
46
|
+
# Written to a temp file by `.options`.
|
|
47
|
+
when "IdentityKey"
|
|
48
|
+
(options[:key_data] ||= []) << value
|
|
49
|
+
options[:keys_only] = true
|
|
50
|
+
when "IdentityFile"
|
|
51
|
+
(options[:keys] ||= []) << value
|
|
52
|
+
options[:keys_only] = true
|
|
53
|
+
when "UserKnownHostsFile"
|
|
54
|
+
options[:user_known_hosts_file] = value.to_s.split
|
|
55
|
+
when "StrictHostKeyChecking"
|
|
56
|
+
options[:verify_host_key] = verify_host_key(value)
|
|
57
|
+
when "ConnectTimeout"
|
|
58
|
+
options[:timeout] = Integer(value)
|
|
59
|
+
when "ServerAliveInterval"
|
|
60
|
+
options[:keepalive] = Integer(value).positive?
|
|
61
|
+
options[:keepalive_interval] = Integer(value)
|
|
62
|
+
when "ServerAliveCountMax"
|
|
63
|
+
options[:keepalive_maxcount] = Integer(value)
|
|
64
|
+
when "LogLevel"
|
|
65
|
+
options[:verbose] = log_level(value)
|
|
66
|
+
else
|
|
67
|
+
raise(ArgumentError,
|
|
68
|
+
"SFTP #each_child does not support the ssh option #{key.inspect}. It supports: #{OPTIONS.join(', ')}")
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def self.verify_host_key(value)
|
|
73
|
+
case value.to_s.downcase
|
|
74
|
+
when "yes", "ask"
|
|
75
|
+
:always
|
|
76
|
+
when "accept-new"
|
|
77
|
+
:accept_new
|
|
78
|
+
when "no", "off"
|
|
79
|
+
:never
|
|
80
|
+
else
|
|
81
|
+
raise(ArgumentError, "Invalid StrictHostKeyChecking value: #{value.inspect}")
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def self.log_level(value)
|
|
86
|
+
case value.to_s.upcase
|
|
87
|
+
when "QUIET", "FATAL"
|
|
88
|
+
:fatal
|
|
89
|
+
when "ERROR"
|
|
90
|
+
:error
|
|
91
|
+
when "INFO", "VERBOSE"
|
|
92
|
+
:info
|
|
93
|
+
when "DEBUG", "DEBUG1", "DEBUG2", "DEBUG3"
|
|
94
|
+
:debug
|
|
95
|
+
else
|
|
96
|
+
raise(ArgumentError, "Invalid LogLevel value: #{value.inspect}")
|
|
97
|
+
end
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
private_class_method :add, :verify_host_key, :log_level
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
end
|
|
@@ -32,6 +32,8 @@ module IOStreams
|
|
|
32
32
|
@before_password_wait_seconds = 2
|
|
33
33
|
@sshpass_wait_seconds = 5
|
|
34
34
|
|
|
35
|
+
autoload :NetSSH, "io_streams/paths/sftp/net_ssh"
|
|
36
|
+
|
|
35
37
|
attr_reader :hostname, :username, :ssh_options, :url, :port
|
|
36
38
|
|
|
37
39
|
# Stream to a remote file over sftp.
|
|
@@ -39,6 +41,11 @@ module IOStreams
|
|
|
39
41
|
# url: [String]
|
|
40
42
|
# "sftp://<host_name>/<file_name>"
|
|
41
43
|
#
|
|
44
|
+
# SECURITY WARNING:
|
|
45
|
+
# A username and password supplied in the url remain part of it, so `#to_s` and `#url`
|
|
46
|
+
# return them, as does any log or error message that includes the path.
|
|
47
|
+
# Supply them with the `username:` and `password:` arguments instead.
|
|
48
|
+
#
|
|
42
49
|
# username: [String]
|
|
43
50
|
# Name of user to login with.
|
|
44
51
|
#
|
|
@@ -63,6 +70,11 @@ module IOStreams
|
|
|
63
70
|
# - Any other options supported by ssh_config.
|
|
64
71
|
# `man ssh_config` to see all available options.
|
|
65
72
|
#
|
|
73
|
+
# `#each_child` lists files with the net-sftp gem instead of the sftp executable, so it only supports
|
|
74
|
+
# these ssh options: HostKey, IdentityKey, IdentityFile, UserKnownHostsFile, StrictHostKeyChecking,
|
|
75
|
+
# ConnectTimeout, ServerAliveInterval, ServerAliveCountMax and LogLevel. Any other option raises
|
|
76
|
+
# ArgumentError.
|
|
77
|
+
#
|
|
66
78
|
# Examples:
|
|
67
79
|
#
|
|
68
80
|
# # Display the contents of a remote file
|
|
@@ -102,7 +114,7 @@ module IOStreams
|
|
|
102
114
|
# Not Ruby 2.5 yet: transform_keys(&:to_s)
|
|
103
115
|
@ssh_options = {}
|
|
104
116
|
ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
|
|
105
|
-
|
|
117
|
+
validate_username!
|
|
106
118
|
|
|
107
119
|
super(uri.path)
|
|
108
120
|
end
|
|
@@ -124,7 +136,9 @@ module IOStreams
|
|
|
124
136
|
|
|
125
137
|
# TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
|
|
126
138
|
|
|
127
|
-
# Search for files on the remote sftp server that match the provided pattern.
|
|
139
|
+
# Search for files on the remote sftp server that match the provided pattern, within this path.
|
|
140
|
+
# When the url does not include a path, for example `sftp://sftp.example.org`, it searches
|
|
141
|
+
# the login directory.
|
|
128
142
|
#
|
|
129
143
|
# The pattern matching works like Net::SFTP::Operations::Dir.glob and Dir.glob
|
|
130
144
|
# Each child also returns attributes that contain the file size, ownership, file dates and other details.
|
|
@@ -137,7 +151,7 @@ module IOStreams
|
|
|
137
151
|
# end
|
|
138
152
|
#
|
|
139
153
|
# Example Output:
|
|
140
|
-
# sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
|
|
154
|
+
# sftp://sftp.example.org/my_files/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
|
|
141
155
|
# :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
|
|
142
156
|
def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
|
|
143
157
|
unless block_given?
|
|
@@ -145,64 +159,95 @@ module IOStreams
|
|
|
145
159
|
case_sensitive: case_sensitive, directories: directories, hidden: hidden)
|
|
146
160
|
end
|
|
147
161
|
|
|
162
|
+
authorize!
|
|
148
163
|
Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
|
|
149
164
|
|
|
150
165
|
flags = ::File::FNM_EXTGLOB
|
|
151
166
|
flags |= ::File::FNM_CASEFOLD unless case_sensitive
|
|
152
167
|
flags |= ::File::FNM_DOTMATCH if hidden
|
|
153
168
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
169
|
+
NetSSH.options(ssh_options, port: port, password: password) do |options|
|
|
170
|
+
Net::SFTP.start(hostname, username, options) do |sftp|
|
|
171
|
+
# Without a path in the url, list the login directory.
|
|
172
|
+
sftp.dir.glob(path.empty? ? "." : path, pattern, flags) do |entry|
|
|
173
|
+
next if !directories && !entry.file?
|
|
157
174
|
|
|
158
|
-
|
|
159
|
-
|
|
175
|
+
child = child_path(entry.name)
|
|
176
|
+
yield(child, entry.attributes.attributes) if allowed_child?(child)
|
|
177
|
+
end
|
|
160
178
|
end
|
|
161
179
|
end
|
|
162
180
|
nil
|
|
163
181
|
end
|
|
164
182
|
|
|
183
|
+
protected
|
|
184
|
+
|
|
185
|
+
attr_writer :url
|
|
186
|
+
|
|
165
187
|
private
|
|
166
188
|
|
|
167
189
|
attr_reader :password
|
|
168
190
|
|
|
191
|
+
# Returns [String] the host, port and path, which is compared against the allowed paths.
|
|
192
|
+
# `.` and `..` are resolved the way the sftp server resolves them.
|
|
193
|
+
def allowed_location
|
|
194
|
+
"sftp://#{hostname.to_s.downcase}:#{port}#{normalize_path(path)}".chomp("/")
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# Usernames are passed to the `sftp` executable, so reject values that it could treat as options.
|
|
198
|
+
def validate_username!
|
|
199
|
+
return if username.nil?
|
|
200
|
+
return unless username.to_s.start_with?("-") || username.to_s.match?(/[[:cntrl:]]/)
|
|
201
|
+
|
|
202
|
+
raise(ArgumentError, "Invalid SFTP username: it cannot start with '-' or contain control characters")
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Set the path directly rather than parsing it as part of a URL, since a file name can contain
|
|
206
|
+
# characters such as `?`, `#`, `+` or `%` that a URL parser would treat as a query or as escapes.
|
|
207
|
+
#
|
|
208
|
+
# The supplied name is relative to this path, or to the login directory when this url has no path.
|
|
209
|
+
def child_path(name)
|
|
210
|
+
server = port == 22 ? "sftp://#{hostname}" : "sftp://#{hostname}:#{port}"
|
|
211
|
+
child = self.class.new(server, username: username, password: password, ssh_options: ssh_options)
|
|
212
|
+
child.path = (path.empty? ? "/#{name}" : ::File.join(path, name)).freeze
|
|
213
|
+
child.url = "#{server}#{child.path}"
|
|
214
|
+
child
|
|
215
|
+
end
|
|
216
|
+
|
|
169
217
|
def stream_reader(&block)
|
|
170
|
-
|
|
171
|
-
sftp_download(path,
|
|
172
|
-
::File.open(
|
|
218
|
+
Utils.private_temp_file("iostreams-sftp-reader") do |file_name|
|
|
219
|
+
sftp_download(path, file_name)
|
|
220
|
+
::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
|
|
173
221
|
end
|
|
174
222
|
end
|
|
175
223
|
|
|
176
224
|
def stream_writer(&block)
|
|
177
|
-
|
|
178
|
-
::File.open(
|
|
179
|
-
sftp_upload(
|
|
180
|
-
|
|
225
|
+
Utils.private_temp_file("iostreams-sftp-writer") do |file_name|
|
|
226
|
+
::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
|
|
227
|
+
sftp_upload(file_name, path)
|
|
228
|
+
::File.size(file_name)
|
|
181
229
|
end
|
|
182
230
|
end
|
|
183
231
|
|
|
184
|
-
# Use sftp
|
|
232
|
+
# Use the sftp executable to download to a local file, via sshpass when a password is supplied
|
|
185
233
|
def sftp_download(remote_file_name, local_file_name)
|
|
186
234
|
with_sftp_args do |args|
|
|
187
235
|
Open3.popen2e(*args) do |writer, reader, waith_thr|
|
|
188
|
-
|
|
189
|
-
|
|
236
|
+
if password
|
|
237
|
+
# Give time for remote sftp server to get ready to accept the password.
|
|
238
|
+
sleep self.class.before_password_wait_seconds
|
|
190
239
|
|
|
191
|
-
|
|
240
|
+
writer.puts password
|
|
192
241
|
|
|
193
|
-
|
|
194
|
-
|
|
242
|
+
# Give time for password to be processed and stdin to be passed to sftp process.
|
|
243
|
+
sleep self.class.sshpass_wait_seconds
|
|
244
|
+
end
|
|
195
245
|
|
|
196
246
|
writer.puts "get #{remote_file_name.inspect} #{local_file_name.inspect}"
|
|
197
247
|
writer.puts "bye"
|
|
198
248
|
writer.close
|
|
199
249
|
out = reader.read.chomp
|
|
200
|
-
unless waith_thr.value.success?
|
|
201
|
-
raise(
|
|
202
|
-
Errors::CommunicationsFailure,
|
|
203
|
-
"Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
|
204
|
-
)
|
|
205
|
-
end
|
|
250
|
+
raise_failure("Download", out) unless waith_thr.value.success?
|
|
206
251
|
|
|
207
252
|
out
|
|
208
253
|
rescue Errno::EPIPE
|
|
@@ -211,10 +256,7 @@ module IOStreams
|
|
|
211
256
|
rescue StandardError
|
|
212
257
|
nil
|
|
213
258
|
end
|
|
214
|
-
|
|
215
|
-
Errors::CommunicationsFailure,
|
|
216
|
-
"Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
|
217
|
-
)
|
|
259
|
+
raise_failure("Download", out)
|
|
218
260
|
end
|
|
219
261
|
end
|
|
220
262
|
end
|
|
@@ -222,19 +264,16 @@ module IOStreams
|
|
|
222
264
|
def sftp_upload(local_file_name, remote_file_name)
|
|
223
265
|
with_sftp_args do |args|
|
|
224
266
|
Open3.popen2e(*args) do |writer, reader, waith_thr|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
267
|
+
if password
|
|
268
|
+
writer.puts(password)
|
|
269
|
+
# Give time for password to be processed and stdin to be passed to sftp process.
|
|
270
|
+
sleep self.class.sshpass_wait_seconds
|
|
271
|
+
end
|
|
228
272
|
writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
|
|
229
273
|
writer.puts "bye"
|
|
230
274
|
writer.close
|
|
231
275
|
out = reader.read.chomp
|
|
232
|
-
unless waith_thr.value.success?
|
|
233
|
-
raise(
|
|
234
|
-
Errors::CommunicationsFailure,
|
|
235
|
-
"Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
|
236
|
-
)
|
|
237
|
-
end
|
|
276
|
+
raise_failure("Upload", out) unless waith_thr.value.success?
|
|
238
277
|
|
|
239
278
|
out
|
|
240
279
|
rescue Errno::EPIPE
|
|
@@ -243,14 +282,21 @@ module IOStreams
|
|
|
243
282
|
rescue StandardError
|
|
244
283
|
nil
|
|
245
284
|
end
|
|
246
|
-
|
|
247
|
-
Errors::CommunicationsFailure,
|
|
248
|
-
"Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
|
249
|
-
)
|
|
285
|
+
raise_failure("Upload", out)
|
|
250
286
|
end
|
|
251
287
|
end
|
|
252
288
|
end
|
|
253
289
|
|
|
290
|
+
# When the server does not prompt for a password, sftp reads the password line as a command
|
|
291
|
+
# and echoes it in its output, so remove it before the output is included in the error.
|
|
292
|
+
def raise_failure(action, out)
|
|
293
|
+
out = out.gsub(password.to_s, "[FILTERED]") if out && !password.to_s.empty?
|
|
294
|
+
raise(
|
|
295
|
+
Errors::CommunicationsFailure,
|
|
296
|
+
"#{action} failed calling #{self.class.sftp_bin}#{" via #{self.class.sshpass_bin}" if password}: #{out}"
|
|
297
|
+
)
|
|
298
|
+
end
|
|
299
|
+
|
|
254
300
|
def with_sftp_args
|
|
255
301
|
return yield sftp_args(ssh_options) if !ssh_options.key?("IdentityKey") && !ssh_options.key?("HostKey")
|
|
256
302
|
|
|
@@ -274,9 +320,9 @@ module IOStreams
|
|
|
274
320
|
end
|
|
275
321
|
|
|
276
322
|
def with_temp_file(options, option, value)
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
::File.
|
|
323
|
+
# sftp requires that private key is only readable by the current user
|
|
324
|
+
Utils.private_temp_file("iostreams-sftp-args", "key") do |file_name|
|
|
325
|
+
::File.binwrite(file_name, value)
|
|
280
326
|
|
|
281
327
|
options[option] = file_name
|
|
282
328
|
yield options
|
|
@@ -284,7 +330,8 @@ module IOStreams
|
|
|
284
330
|
end
|
|
285
331
|
|
|
286
332
|
def sftp_args(ssh_options)
|
|
287
|
-
|
|
333
|
+
# sshpass is only needed to supply the password to sftp.
|
|
334
|
+
args = password ? [self.class.sshpass_bin, self.class.sftp_bin] : [self.class.sftp_bin]
|
|
288
335
|
# Force sftp to use the password when supplied,
|
|
289
336
|
# and stop sftp from prompting for a password when none was supplied.
|
|
290
337
|
if password
|
|
@@ -303,19 +350,12 @@ module IOStreams
|
|
|
303
350
|
ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
|
|
304
351
|
args << "-b"
|
|
305
352
|
args << "-"
|
|
353
|
+
# Stop sftp from treating the destination as an option.
|
|
354
|
+
args << "--"
|
|
306
355
|
args << "#{username}@#{hostname}"
|
|
307
356
|
args
|
|
308
357
|
end
|
|
309
358
|
|
|
310
|
-
def build_ssh_options
|
|
311
|
-
options = ssh_options.dup
|
|
312
|
-
options[:logger] ||= IOStreams.logger if IOStreams.logger
|
|
313
|
-
options[:port] ||= port
|
|
314
|
-
options[:max_pkt_size] ||= 65_536
|
|
315
|
-
options[:password] ||= @password
|
|
316
|
-
options
|
|
317
|
-
end
|
|
318
|
-
|
|
319
359
|
def map_log_level
|
|
320
360
|
level = IOStreams.logger&.level
|
|
321
361
|
case level
|