iostreams 2.0.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +5 -22
  3. data/Rakefile +45 -0
  4. data/docs/CLAUDE.md +9 -0
  5. data/docs/config.md +157 -0
  6. data/docs/copy_files.md +75 -0
  7. data/docs/extensions.md +111 -0
  8. data/docs/formats.md +188 -0
  9. data/docs/index.md +388 -0
  10. data/docs/path.md +652 -0
  11. data/docs/pgp.md +436 -0
  12. data/docs/streams.md +337 -0
  13. data/docs/tutorial.md +483 -0
  14. data/docs/upgrading.md +217 -0
  15. data/lib/io_streams/builder.rb +62 -2
  16. data/lib/io_streams/bzip2/reader.rb +25 -2
  17. data/lib/io_streams/bzip2/writer.rb +26 -2
  18. data/lib/io_streams/encode/reader.rb +4 -0
  19. data/lib/io_streams/encode/writer.rb +4 -0
  20. data/lib/io_streams/errors.rb +4 -0
  21. data/lib/io_streams/gzip/reader.rb +4 -0
  22. data/lib/io_streams/gzip/writer.rb +11 -2
  23. data/lib/io_streams/io_streams.rb +111 -1
  24. data/lib/io_streams/line/reader.rb +7 -2
  25. data/lib/io_streams/path.rb +115 -6
  26. data/lib/io_streams/paths/file.rb +47 -1
  27. data/lib/io_streams/paths/http.rb +45 -4
  28. data/lib/io_streams/paths/s3.rb +66 -15
  29. data/lib/io_streams/paths/sftp/net_ssh.rb +104 -0
  30. data/lib/io_streams/paths/sftp.rb +97 -57
  31. data/lib/io_streams/pgp/reader.rb +42 -2
  32. data/lib/io_streams/pgp/writer.rb +26 -6
  33. data/lib/io_streams/pgp.rb +78 -21
  34. data/lib/io_streams/reader.rb +10 -1
  35. data/lib/io_streams/record/reader.rb +72 -2
  36. data/lib/io_streams/stream.rb +12 -7
  37. data/lib/io_streams/symmetric_encryption/reader.rb +4 -0
  38. data/lib/io_streams/symmetric_encryption/writer.rb +4 -0
  39. data/lib/io_streams/tabular/header.rb +31 -4
  40. data/lib/io_streams/tabular/parser/base.rb +10 -0
  41. data/lib/io_streams/tabular/parser/csv.rb +5 -0
  42. data/lib/io_streams/tabular/parser/fixed.rb +3 -1
  43. data/lib/io_streams/tabular/parser/psv.rb +6 -2
  44. data/lib/io_streams/utils.rb +31 -0
  45. data/lib/io_streams/version.rb +1 -1
  46. data/lib/io_streams/writer.rb +10 -1
  47. data/lib/io_streams/xlsx/reader.rb +5 -1
  48. data/lib/io_streams/zip/reader.rb +4 -0
  49. data/lib/io_streams/zip/writer.rb +4 -0
  50. metadata +24 -7
@@ -18,6 +18,11 @@ module IOStreams
18
18
  # Full url showing all the optional elements that can be set via the url:
19
19
  # https://username:password@hostname/path/file_name
20
20
  #
21
+ # SECURITY WARNING:
22
+ # A username and password supplied in the url remain part of it, so `#to_s` and `#url`
23
+ # return them, as does any log or error message that includes the path.
24
+ # Supply them with the `username:` and `password:` arguments instead.
25
+ #
21
26
  # username: [String]
22
27
  # When supplied, basic authentication is used with the username and password.
23
28
  #
@@ -108,7 +113,9 @@ module IOStreams
108
113
  request.basic_auth(username, password) if username && same_origin?(uri)
109
114
 
110
115
  http.request(request) do |response|
111
- raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{uri}") if response.is_a?(Net::HTTPNotFound)
116
+ if response.is_a?(Net::HTTPNotFound)
117
+ raise(IOStreams::Errors::CommunicationsFailure, "Invalid URL: #{without_credentials(uri)}")
118
+ end
112
119
  if response.is_a?(Net::HTTPUnauthorized)
113
120
  raise(IOStreams::Errors::CommunicationsFailure, "Authorization Required: Invalid :username or :password.")
114
121
  end
@@ -117,7 +124,10 @@ module IOStreams
117
124
  raise(IOStreams::Errors::CommunicationsFailure, "Too many redirects") if http_redirect_count < 1
118
125
 
119
126
  location = response["location"]
120
- raise(IOStreams::Errors::CommunicationsFailure, "Redirect missing location header: #{uri}") unless location
127
+ unless location
128
+ raise(IOStreams::Errors::CommunicationsFailure,
129
+ "Redirect missing location header: #{without_credentials(uri)}")
130
+ end
121
131
 
122
132
  # Resolve relative redirects against the current uri.
123
133
  new_uri = uri.merge(location)
@@ -129,7 +139,7 @@ module IOStreams
129
139
  end
130
140
 
131
141
  # Since Net::HTTP download only supports a push stream, write it to a tempfile first.
132
- Utils.temp_file_name("iostreams_http") do |file_name|
142
+ Utils.private_temp_file("iostreams_http") do |file_name|
133
143
  download_to_file(response, file_name)
134
144
  # Return a read stream
135
145
  result = ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
@@ -141,15 +151,46 @@ module IOStreams
141
151
 
142
152
  # Validate that the host may be contacted, and that the scheme is still http(s)
143
153
  # after following a redirect.
154
+ #
155
+ # A redirect must also be within the allowed paths, see `IOStreams.add_allowed_path`.
144
156
  def validate_uri!(uri)
145
157
  unless %w[http https].include?(uri.scheme)
146
- raise(IOStreams::Errors::CommunicationsFailure, "Invalid redirect, only http and https are supported: #{uri}")
158
+ raise(IOStreams::Errors::CommunicationsFailure,
159
+ "Invalid redirect, only http and https are supported: #{without_credentials(uri)}")
147
160
  end
161
+ authorize_location!(http_location(uri)) unless IOStreams.allowed_paths.empty?
148
162
  return if allow_hosts.nil? || allow_hosts.include?(uri.hostname)
149
163
 
150
164
  raise(IOStreams::Errors::CommunicationsFailure, "Host not in the allowed list of hosts: #{uri.hostname}")
151
165
  end
152
166
 
167
+ # Returns [String] the scheme, host, port and path of the url, which is compared against the allowed paths.
168
+ def allowed_location
169
+ http_location(original_uri)
170
+ end
171
+
172
+ # Returns [String] the scheme, host, port and path of the supplied uri, without any credentials or query.
173
+ #
174
+ # The path is decoded and `.` and `..` resolved, the way most web servers resolve them, so that for
175
+ # example `%2e%2e` cannot be used to leave an allowed path. A backslash is treated as a `/`.
176
+ def http_location(uri)
177
+ raise(Errors::AccessDenied, "Access denied: #{without_credentials(uri)} has no host") if uri.host.to_s.empty?
178
+
179
+ path = URI.decode_uri_component(uri.path).tr("\\", "/")
180
+ "#{uri.scheme}://#{uri.host.downcase}:#{uri.port}#{normalize_path(path)}".chomp("/")
181
+ rescue ArgumentError => e
182
+ raise(Errors::AccessDenied, "Access denied to #{without_credentials(uri)}: #{e.message}")
183
+ end
184
+
185
+ # Returns [String] the uri without any user name or password, for use in error messages.
186
+ def without_credentials(uri)
187
+ return uri.to_s unless uri.user
188
+
189
+ uri = uri.dup
190
+ uri.user = nil
191
+ uri.to_s
192
+ end
193
+
153
194
  def same_origin?(uri)
154
195
  original = original_uri
155
196
  uri.scheme == original.scheme && uri.hostname == original.hostname && uri.port == original.port
@@ -21,6 +21,15 @@ module IOStreams
21
21
  # s3://my-bucket-name/file_name.txt
22
22
  # s3://my-bucket-name/some_path/file_name.csv
23
23
  #
24
+ # Any query string in the url is added to the S3 request parameters, for example:
25
+ # s3://my-bucket-name/file_name.csv?acl=bucket-owner-full-control
26
+ #
27
+ # SECURITY WARNING:
28
+ # Do not interpolate untrusted file names into the url, since a name such as
29
+ # `file.csv?acl=public-read` would set request parameters.
30
+ # Instead join untrusted names onto the path, which does not parse them as a query:
31
+ # IOStreams.path("s3://my-bucket-name/uploads").join(untrusted_name)
32
+ #
24
33
  # access_key_id: [String]
25
34
  # AWS Access Key Id to use to access this bucket.
26
35
  #
@@ -185,6 +194,7 @@ module IOStreams
185
194
  end
186
195
 
187
196
  def delete
197
+ authorize!
188
198
  client.delete_object(bucket: bucket_name, key: path)
189
199
  self
190
200
  rescue Aws::S3::Errors::NotFound
@@ -192,6 +202,7 @@ module IOStreams
192
202
  end
193
203
 
194
204
  def exist?
205
+ authorize!
195
206
  client.head_object(bucket: bucket_name, key: path)
196
207
  true
197
208
  rescue Aws::S3::Errors::NotFound
@@ -216,6 +227,8 @@ module IOStreams
216
227
  target = IOStreams.new(target_path)
217
228
  return super(target, convert: convert, **args) unless target.is_a?(self.class)
218
229
 
230
+ authorize!
231
+ target.authorize!
219
232
  source_name = ::File.join(bucket_name, path)
220
233
  client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
221
234
  target
@@ -230,6 +243,8 @@ module IOStreams
230
243
  return super(source, convert: convert, **args)
231
244
  end
232
245
 
246
+ authorize!
247
+ source.authorize!
233
248
  source_name = ::File.join(source.bucket_name, source.path)
234
249
  client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
235
250
  end
@@ -244,6 +259,7 @@ module IOStreams
244
259
  end
245
260
 
246
261
  def size
262
+ authorize!
247
263
  client.head_object(bucket: bucket_name, key: path).content_length
248
264
  rescue Aws::S3::Errors::NotFound
249
265
  nil
@@ -254,7 +270,7 @@ module IOStreams
254
270
  # Read from AWS S3 file.
255
271
  def stream_reader(&block)
256
272
  # Since S3 download only supports a push stream, write it to a tempfile first.
257
- Utils.temp_file_name("iostreams_s3") do |file_name|
273
+ Utils.private_temp_file("iostreams_s3") do |file_name|
258
274
  read_file(file_name)
259
275
 
260
276
  ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
@@ -263,6 +279,7 @@ module IOStreams
263
279
 
264
280
  # Shortcut method if caller has a filename already with no other streams applied:
265
281
  def read_file(file_name)
282
+ authorize!
266
283
  ::File.open(file_name, "wb") do |file|
267
284
  client.get_object(options.merge(response_target: file, bucket: bucket_name, key: path))
268
285
  end
@@ -277,7 +294,7 @@ module IOStreams
277
294
  # aborted.
278
295
  def stream_writer(&block)
279
296
  # Since S3 upload only supports a pull stream, write it to a tempfile first.
280
- Utils.temp_file_name("iostreams_s3") do |file_name|
297
+ Utils.private_temp_file("iostreams_s3") do |file_name|
281
298
  result = ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
282
299
 
283
300
  # Upload file only once all data has been written to it
@@ -288,6 +305,7 @@ module IOStreams
288
305
 
289
306
  # Shortcut method if caller has a filename already with no other streams applied:
290
307
  def write_file(file_name)
308
+ authorize!
291
309
  if ::File.size(file_name) > MULTIPART_UPLOAD_SIZE
292
310
  # Use multipart file upload
293
311
  s3 = Aws::S3::Resource.new(client: client)
@@ -308,29 +326,26 @@ module IOStreams
308
326
  case_sensitive: case_sensitive, directories: directories, hidden: hidden)
309
327
  end
310
328
 
329
+ authorize!
311
330
  matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
312
331
 
313
332
  # When the pattern includes an exact file name without any pattern characters
314
333
  if matcher.pattern.nil?
315
- yield(matcher.path) if matcher.path.exist?
334
+ yield(matcher.path) if allowed_child?(matcher.path) && matcher.path.exist?
316
335
  return
317
336
  end
318
337
 
319
338
  prefix = Utils::URI.new(matcher.path.to_s).path.sub(%r{\A/}, "")
320
- token = nil
321
- loop do
322
- # Fetches upto 1,000 entries at a time
323
- resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
324
- resp.contents.each do |object|
325
- next if !directories && object.key.end_with?("/")
339
+ each_object(prefix) do |name, object|
340
+ next if !directories && object.key.end_with?("/")
326
341
 
327
- file_name = ::File.join("s3://", resp.name, object.key)
328
- next unless matcher.match?(file_name)
342
+ file_name = ::File.join("s3://", name, object.key)
343
+ next unless matcher.match?(file_name)
329
344
 
330
- yield(self.class.new(file_name), object.to_h)
331
- end
332
- token = resp.next_continuation_token
333
- break if token.nil?
345
+ child = child_path(name, object.key)
346
+ next unless allowed_child?(child)
347
+
348
+ yield(child, object.to_h)
334
349
  end
335
350
  nil
336
351
  end
@@ -344,6 +359,42 @@ module IOStreams
344
359
  def client
345
360
  @client ||= ::Aws::S3::Client.new(@client_options)
346
361
  end
362
+
363
+ private
364
+
365
+ # Yields the bucket name and each object in the bucket whose key starts with the supplied prefix.
366
+ def each_object(prefix)
367
+ token = nil
368
+ loop do
369
+ # Fetches upto 1,000 entries at a time
370
+ resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
371
+ resp.contents.each { |object| yield(resp.name, object) }
372
+ token = resp.next_continuation_token
373
+ break if token.nil?
374
+ end
375
+ end
376
+
377
+ # Returns [String] the bucket and key, which is compared against the allowed paths.
378
+ #
379
+ # S3 treats `.` and `..` in a key as ordinary characters, but other services that implement the
380
+ # S3 API may resolve them, so keys containing them are denied.
381
+ def allowed_location
382
+ if path.split("/").intersect?([".", ".."])
383
+ raise(Errors::AccessDenied, "Access denied to #{self}: '.' and '..' are not allowed in S3 keys")
384
+ end
385
+
386
+ to_s.sub(%r{/+\z}, "")
387
+ end
388
+
389
+ # Set the key directly rather than parsing it as part of a URL, since a key can contain
390
+ # characters such as `?`, `+` or `%` that a URL parser would treat as a query or as escapes.
391
+ #
392
+ # The child uses this path's client, so that it has the same credentials and region.
393
+ def child_path(bucket_name, key)
394
+ child = self.class.new("s3://#{bucket_name}", client: client)
395
+ child.path = key.dup.freeze
396
+ child
397
+ end
347
398
  end
348
399
  end
349
400
  end
@@ -0,0 +1,104 @@
1
+ module IOStreams
2
+ module Paths
3
+ class SFTP < IOStreams::Path
4
+ # Translates the ssh options supplied for the sftp executable into options for net-ssh,
5
+ # which `SFTP#each_child` uses to list files instead of the sftp executable.
6
+ #
7
+ # The net-ssh options match those that SFTP passes to the sftp executable.
8
+ module NetSSH
9
+ # The ssh options that net-ssh supports.
10
+ OPTIONS = %w[
11
+ HostKey IdentityKey IdentityFile UserKnownHostsFile StrictHostKeyChecking
12
+ ConnectTimeout ServerAliveInterval ServerAliveCountMax LogLevel
13
+ ].freeze
14
+
15
+ # Yields [Hash] the net-ssh options.
16
+ #
17
+ # Raises [ArgumentError] when `ssh_options` includes an option that is not in `OPTIONS`.
18
+ def self.options(ssh_options, port:, password:)
19
+ options = {port: port, max_pkt_size: 65_536, non_interactive: true}
20
+ options[:logger] = IOStreams.logger if IOStreams.logger
21
+ # Like the sftp executable, only use the password when one is supplied, and otherwise only public keys.
22
+ if password
23
+ options[:password] = password
24
+ options[:auth_methods] = %w[password keyboard-interactive]
25
+ else
26
+ options[:auth_methods] = %w[publickey]
27
+ end
28
+ # Like the sftp executable, which uses `StrictHostKeyChecking=yes`, instead of the
29
+ # net-ssh default of trusting a host key the first time it is seen.
30
+ options[:verify_host_key] = :always
31
+
32
+ ssh_options.each_pair { |key, value| add(options, key, value) }
33
+ return yield(options) unless ssh_options.key?("HostKey")
34
+
35
+ # Like the sftp executable, the host key replaces the user's known_hosts file.
36
+ Utils.private_temp_file("iostreams-sftp-known-hosts") do |file_name|
37
+ ::File.binwrite(file_name, ssh_options["HostKey"])
38
+ options[:user_known_hosts_file] = [file_name]
39
+ yield(options)
40
+ end
41
+ end
42
+
43
+ def self.add(options, key, value)
44
+ case key
45
+ when "HostKey"
46
+ # Written to a temp file by `.options`.
47
+ when "IdentityKey"
48
+ (options[:key_data] ||= []) << value
49
+ options[:keys_only] = true
50
+ when "IdentityFile"
51
+ (options[:keys] ||= []) << value
52
+ options[:keys_only] = true
53
+ when "UserKnownHostsFile"
54
+ options[:user_known_hosts_file] = value.to_s.split
55
+ when "StrictHostKeyChecking"
56
+ options[:verify_host_key] = verify_host_key(value)
57
+ when "ConnectTimeout"
58
+ options[:timeout] = Integer(value)
59
+ when "ServerAliveInterval"
60
+ options[:keepalive] = Integer(value).positive?
61
+ options[:keepalive_interval] = Integer(value)
62
+ when "ServerAliveCountMax"
63
+ options[:keepalive_maxcount] = Integer(value)
64
+ when "LogLevel"
65
+ options[:verbose] = log_level(value)
66
+ else
67
+ raise(ArgumentError,
68
+ "SFTP #each_child does not support the ssh option #{key.inspect}. It supports: #{OPTIONS.join(', ')}")
69
+ end
70
+ end
71
+
72
+ def self.verify_host_key(value)
73
+ case value.to_s.downcase
74
+ when "yes", "ask"
75
+ :always
76
+ when "accept-new"
77
+ :accept_new
78
+ when "no", "off"
79
+ :never
80
+ else
81
+ raise(ArgumentError, "Invalid StrictHostKeyChecking value: #{value.inspect}")
82
+ end
83
+ end
84
+
85
+ def self.log_level(value)
86
+ case value.to_s.upcase
87
+ when "QUIET", "FATAL"
88
+ :fatal
89
+ when "ERROR"
90
+ :error
91
+ when "INFO", "VERBOSE"
92
+ :info
93
+ when "DEBUG", "DEBUG1", "DEBUG2", "DEBUG3"
94
+ :debug
95
+ else
96
+ raise(ArgumentError, "Invalid LogLevel value: #{value.inspect}")
97
+ end
98
+ end
99
+
100
+ private_class_method :add, :verify_host_key, :log_level
101
+ end
102
+ end
103
+ end
104
+ end
@@ -32,6 +32,8 @@ module IOStreams
32
32
  @before_password_wait_seconds = 2
33
33
  @sshpass_wait_seconds = 5
34
34
 
35
+ autoload :NetSSH, "io_streams/paths/sftp/net_ssh"
36
+
35
37
  attr_reader :hostname, :username, :ssh_options, :url, :port
36
38
 
37
39
  # Stream to a remote file over sftp.
@@ -39,6 +41,11 @@ module IOStreams
39
41
  # url: [String]
40
42
  # "sftp://<host_name>/<file_name>"
41
43
  #
44
+ # SECURITY WARNING:
45
+ # A username and password supplied in the url remain part of it, so `#to_s` and `#url`
46
+ # return them, as does any log or error message that includes the path.
47
+ # Supply them with the `username:` and `password:` arguments instead.
48
+ #
42
49
  # username: [String]
43
50
  # Name of user to login with.
44
51
  #
@@ -63,6 +70,11 @@ module IOStreams
63
70
  # - Any other options supported by ssh_config.
64
71
  # `man ssh_config` to see all available options.
65
72
  #
73
+ # `#each_child` lists files with the net-sftp gem instead of the sftp executable, so it only supports
74
+ # these ssh options: HostKey, IdentityKey, IdentityFile, UserKnownHostsFile, StrictHostKeyChecking,
75
+ # ConnectTimeout, ServerAliveInterval, ServerAliveCountMax and LogLevel. Any other option raises
76
+ # ArgumentError.
77
+ #
66
78
  # Examples:
67
79
  #
68
80
  # # Display the contents of a remote file
@@ -102,7 +114,7 @@ module IOStreams
102
114
  # Not Ruby 2.5 yet: transform_keys(&:to_s)
103
115
  @ssh_options = {}
104
116
  ssh_options.each_pair { |key, value| @ssh_options[key.to_s] = value }
105
- @ssh_options.merge(uri.query) if uri.query
117
+ validate_username!
106
118
 
107
119
  super(uri.path)
108
120
  end
@@ -124,7 +136,9 @@ module IOStreams
124
136
 
125
137
  # TODO: Add #copy_from shortcut to detect when a file is supplied that does not require conversion.
126
138
 
127
- # Search for files on the remote sftp server that match the provided pattern.
139
+ # Search for files on the remote sftp server that match the provided pattern, within this path.
140
+ # When the url does not include a path, for example `sftp://sftp.example.org`, it searches
141
+ # the login directory.
128
142
  #
129
143
  # The pattern matching works like Net::SFTP::Operations::Dir.glob and Dir.glob
130
144
  # Each child also returns attributes that contain the file size, ownership, file dates and other details.
@@ -137,7 +151,7 @@ module IOStreams
137
151
  # end
138
152
  #
139
153
  # Example Output:
140
- # sftp://sftp.example.org/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
154
+ # sftp://sftp.example.org/my_files/a/b/c/test.txt {:type=>1, :size=>37, :owner=>"test_owner", :group=>"test_group",
141
155
  # :permissions=>420, :atime=>1572378136, :mtime=>1572378136, :link_count=>1, :extended=>{}}
142
156
  def each_child(pattern = "*", case_sensitive: true, directories: false, hidden: false)
143
157
  unless block_given?
@@ -145,64 +159,95 @@ module IOStreams
145
159
  case_sensitive: case_sensitive, directories: directories, hidden: hidden)
146
160
  end
147
161
 
162
+ authorize!
148
163
  Utils.load_soft_dependency("net-sftp", "SFTP glob capability", "net/sftp") unless defined?(Net::SFTP)
149
164
 
150
165
  flags = ::File::FNM_EXTGLOB
151
166
  flags |= ::File::FNM_CASEFOLD unless case_sensitive
152
167
  flags |= ::File::FNM_DOTMATCH if hidden
153
168
 
154
- Net::SFTP.start(hostname, username, build_ssh_options) do |sftp|
155
- sftp.dir.glob(".", pattern, flags) do |path|
156
- next if !directories && !path.file?
169
+ NetSSH.options(ssh_options, port: port, password: password) do |options|
170
+ Net::SFTP.start(hostname, username, options) do |sftp|
171
+ # Without a path in the url, list the login directory.
172
+ sftp.dir.glob(path.empty? ? "." : path, pattern, flags) do |entry|
173
+ next if !directories && !entry.file?
157
174
 
158
- new_path = self.class.new("sftp://#{hostname}/#{path.name}", username: username, password: password, **ssh_options)
159
- yield(new_path, path.attributes.attributes)
175
+ child = child_path(entry.name)
176
+ yield(child, entry.attributes.attributes) if allowed_child?(child)
177
+ end
160
178
  end
161
179
  end
162
180
  nil
163
181
  end
164
182
 
183
+ protected
184
+
185
+ attr_writer :url
186
+
165
187
  private
166
188
 
167
189
  attr_reader :password
168
190
 
191
+ # Returns [String] the host, port and path, which is compared against the allowed paths.
192
+ # `.` and `..` are resolved the way the sftp server resolves them.
193
+ def allowed_location
194
+ "sftp://#{hostname.to_s.downcase}:#{port}#{normalize_path(path)}".chomp("/")
195
+ end
196
+
197
+ # Usernames are passed to the `sftp` executable, so reject values that it could treat as options.
198
+ def validate_username!
199
+ return if username.nil?
200
+ return unless username.to_s.start_with?("-") || username.to_s.match?(/[[:cntrl:]]/)
201
+
202
+ raise(ArgumentError, "Invalid SFTP username: it cannot start with '-' or contain control characters")
203
+ end
204
+
205
+ # Set the path directly rather than parsing it as part of a URL, since a file name can contain
206
+ # characters such as `?`, `#`, `+` or `%` that a URL parser would treat as a query or as escapes.
207
+ #
208
+ # The supplied name is relative to this path, or to the login directory when this url has no path.
209
+ def child_path(name)
210
+ server = port == 22 ? "sftp://#{hostname}" : "sftp://#{hostname}:#{port}"
211
+ child = self.class.new(server, username: username, password: password, ssh_options: ssh_options)
212
+ child.path = (path.empty? ? "/#{name}" : ::File.join(path, name)).freeze
213
+ child.url = "#{server}#{child.path}"
214
+ child
215
+ end
216
+
169
217
  def stream_reader(&block)
170
- IOStreams.temp_file("iostreams-sftp-reader") do |temp_file|
171
- sftp_download(path, temp_file.to_s)
172
- ::File.open(temp_file.to_s, "rb") { |io| builder.reader(io, &block) }
218
+ Utils.private_temp_file("iostreams-sftp-reader") do |file_name|
219
+ sftp_download(path, file_name)
220
+ ::File.open(file_name, "rb") { |io| builder.reader(io, &block) }
173
221
  end
174
222
  end
175
223
 
176
224
  def stream_writer(&block)
177
- IOStreams.temp_file("iostreams-sftp-writer") do |temp_file|
178
- ::File.open(temp_file.to_s, "wb") { |io| builder.writer(io, &block) }
179
- sftp_upload(temp_file.to_s, path)
180
- temp_file.size
225
+ Utils.private_temp_file("iostreams-sftp-writer") do |file_name|
226
+ ::File.open(file_name, "wb") { |io| builder.writer(io, &block) }
227
+ sftp_upload(file_name, path)
228
+ ::File.size(file_name)
181
229
  end
182
230
  end
183
231
 
184
- # Use sftp and sshpass executables to download to a local file
232
+ # Use the sftp executable to download to a local file, via sshpass when a password is supplied
185
233
  def sftp_download(remote_file_name, local_file_name)
186
234
  with_sftp_args do |args|
187
235
  Open3.popen2e(*args) do |writer, reader, waith_thr|
188
- # Give time for remote sftp server to get ready to accept the password.
189
- sleep self.class.before_password_wait_seconds
236
+ if password
237
+ # Give time for remote sftp server to get ready to accept the password.
238
+ sleep self.class.before_password_wait_seconds
190
239
 
191
- writer.puts password
240
+ writer.puts password
192
241
 
193
- # Give time for password to be processed and stdin to be passed to sftp process.
194
- sleep self.class.sshpass_wait_seconds
242
+ # Give time for password to be processed and stdin to be passed to sftp process.
243
+ sleep self.class.sshpass_wait_seconds
244
+ end
195
245
 
196
246
  writer.puts "get #{remote_file_name.inspect} #{local_file_name.inspect}"
197
247
  writer.puts "bye"
198
248
  writer.close
199
249
  out = reader.read.chomp
200
- unless waith_thr.value.success?
201
- raise(
202
- Errors::CommunicationsFailure,
203
- "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
204
- )
205
- end
250
+ raise_failure("Download", out) unless waith_thr.value.success?
206
251
 
207
252
  out
208
253
  rescue Errno::EPIPE
@@ -211,10 +256,7 @@ module IOStreams
211
256
  rescue StandardError
212
257
  nil
213
258
  end
214
- raise(
215
- Errors::CommunicationsFailure,
216
- "Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
217
- )
259
+ raise_failure("Download", out)
218
260
  end
219
261
  end
220
262
  end
@@ -222,19 +264,16 @@ module IOStreams
222
264
  def sftp_upload(local_file_name, remote_file_name)
223
265
  with_sftp_args do |args|
224
266
  Open3.popen2e(*args) do |writer, reader, waith_thr|
225
- writer.puts(password) if password
226
- # Give time for password to be processed and stdin to be passed to sftp process.
227
- sleep self.class.sshpass_wait_seconds
267
+ if password
268
+ writer.puts(password)
269
+ # Give time for password to be processed and stdin to be passed to sftp process.
270
+ sleep self.class.sshpass_wait_seconds
271
+ end
228
272
  writer.puts "put #{local_file_name.inspect} #{remote_file_name.inspect}"
229
273
  writer.puts "bye"
230
274
  writer.close
231
275
  out = reader.read.chomp
232
- unless waith_thr.value.success?
233
- raise(
234
- Errors::CommunicationsFailure,
235
- "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
236
- )
237
- end
276
+ raise_failure("Upload", out) unless waith_thr.value.success?
238
277
 
239
278
  out
240
279
  rescue Errno::EPIPE
@@ -243,14 +282,21 @@ module IOStreams
243
282
  rescue StandardError
244
283
  nil
245
284
  end
246
- raise(
247
- Errors::CommunicationsFailure,
248
- "Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
249
- )
285
+ raise_failure("Upload", out)
250
286
  end
251
287
  end
252
288
  end
253
289
 
290
+ # When the server does not prompt for a password, sftp reads the password line as a command
291
+ # and echoes it in its output, so remove it before the output is included in the error.
292
+ def raise_failure(action, out)
293
+ out = out.gsub(password.to_s, "[FILTERED]") if out && !password.to_s.empty?
294
+ raise(
295
+ Errors::CommunicationsFailure,
296
+ "#{action} failed calling #{self.class.sftp_bin}#{" via #{self.class.sshpass_bin}" if password}: #{out}"
297
+ )
298
+ end
299
+
254
300
  def with_sftp_args
255
301
  return yield sftp_args(ssh_options) if !ssh_options.key?("IdentityKey") && !ssh_options.key?("HostKey")
256
302
 
@@ -274,9 +320,9 @@ module IOStreams
274
320
  end
275
321
 
276
322
  def with_temp_file(options, option, value)
277
- Utils.temp_file_name("iostreams-sftp-args", "key") do |file_name|
278
- # sftp requires that private key is only readable by the current user
279
- ::File.open(file_name, "wb", 0o600) { |io| io.write(value) }
323
+ # sftp requires that private key is only readable by the current user
324
+ Utils.private_temp_file("iostreams-sftp-args", "key") do |file_name|
325
+ ::File.binwrite(file_name, value)
280
326
 
281
327
  options[option] = file_name
282
328
  yield options
@@ -284,7 +330,8 @@ module IOStreams
284
330
  end
285
331
 
286
332
  def sftp_args(ssh_options)
287
- args = [self.class.sshpass_bin, self.class.sftp_bin]
333
+ # sshpass is only needed to supply the password to sftp.
334
+ args = password ? [self.class.sshpass_bin, self.class.sftp_bin] : [self.class.sftp_bin]
288
335
  # Force sftp to use the password when supplied,
289
336
  # and stop sftp from prompting for a password when none was supplied.
290
337
  if password
@@ -303,19 +350,12 @@ module IOStreams
303
350
  ssh_options.each_pair { |key, value| args << "-o#{key}=#{value}" }
304
351
  args << "-b"
305
352
  args << "-"
353
+ # Stop sftp from treating the destination as an option.
354
+ args << "--"
306
355
  args << "#{username}@#{hostname}"
307
356
  args
308
357
  end
309
358
 
310
- def build_ssh_options
311
- options = ssh_options.dup
312
- options[:logger] ||= IOStreams.logger if IOStreams.logger
313
- options[:port] ||= port
314
- options[:max_pkt_size] ||= 65_536
315
- options[:password] ||= @password
316
- options
317
- end
318
-
319
359
  def map_log_level
320
360
  level = IOStreams.logger&.level
321
361
  case level