iostreams 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/docs/config.md +13 -9
  3. data/docs/copy_files.md +20 -2
  4. data/docs/extensions.md +18 -4
  5. data/docs/formats.md +644 -16
  6. data/docs/path.md +18 -2
  7. data/docs/streams.md +54 -9
  8. data/docs/tutorial.md +14 -2
  9. data/docs/upgrading.md +470 -23
  10. data/lib/io_streams/builder/deprecations.rb +250 -0
  11. data/lib/io_streams/builder.rb +256 -39
  12. data/lib/io_streams/bzip2/reader.rb +18 -2
  13. data/lib/io_streams/bzip2/writer.rb +11 -2
  14. data/lib/io_streams/deprecation/methods.rb +51 -0
  15. data/lib/io_streams/deprecation/once.rb +27 -0
  16. data/lib/io_streams/deprecation/option_names.rb +26 -0
  17. data/lib/io_streams/deprecation/words.rb +42 -0
  18. data/lib/io_streams/deprecation.rb +259 -0
  19. data/lib/io_streams/encode/reader.rb +54 -3
  20. data/lib/io_streams/encode/writer.rb +19 -2
  21. data/lib/io_streams/encode.rb +99 -0
  22. data/lib/io_streams/errors.rb +51 -0
  23. data/lib/io_streams/gzip/reader.rb +11 -0
  24. data/lib/io_streams/io_streams.rb +365 -35
  25. data/lib/io_streams/line/reader.rb +2 -1
  26. data/lib/io_streams/line/utf8_reader.rb +23 -0
  27. data/lib/io_streams/path.rb +125 -35
  28. data/lib/io_streams/paths/file.rb +72 -9
  29. data/lib/io_streams/paths/http.rb +48 -2
  30. data/lib/io_streams/paths/matcher.rb +6 -3
  31. data/lib/io_streams/paths/s3.rb +180 -9
  32. data/lib/io_streams/paths/sftp.rb +47 -3
  33. data/lib/io_streams/paths.rb +23 -0
  34. data/lib/io_streams/pgp/reader.rb +18 -0
  35. data/lib/io_streams/pgp/writer.rb +16 -0
  36. data/lib/io_streams/pgp.rb +16 -2
  37. data/lib/io_streams/reader.rb +44 -0
  38. data/lib/io_streams/record/reader.rb +59 -68
  39. data/lib/io_streams/record/writer.rb +17 -10
  40. data/lib/io_streams/row/header_row.rb +21 -0
  41. data/lib/io_streams/row/reader.rb +52 -15
  42. data/lib/io_streams/row/writer.rb +64 -13
  43. data/lib/io_streams/schema/columns.rb +456 -0
  44. data/lib/io_streams/schema/constraints.rb +99 -0
  45. data/lib/io_streams/schema/field.rb +967 -0
  46. data/lib/io_streams/schema/json_number.rb +56 -0
  47. data/lib/io_streams/schema/keys.rb +78 -0
  48. data/lib/io_streams/schema/pattern.rb +257 -0
  49. data/lib/io_streams/schema/source.rb +31 -0
  50. data/lib/io_streams/schema.rb +616 -0
  51. data/lib/io_streams/stream.rb +836 -67
  52. data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
  53. data/lib/io_streams/tabular/header.rb +29 -0
  54. data/lib/io_streams/tabular/header_mapping.rb +332 -0
  55. data/lib/io_streams/tabular/mapping.rb +165 -0
  56. data/lib/io_streams/tabular/parser/array.rb +5 -0
  57. data/lib/io_streams/tabular/parser/base.rb +24 -0
  58. data/lib/io_streams/tabular/parser/fixed.rb +197 -0
  59. data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
  60. data/lib/io_streams/tabular/parser/hash.rb +10 -0
  61. data/lib/io_streams/tabular/parser/json.rb +17 -0
  62. data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
  63. data/lib/io_streams/tabular/schema_mapping.rb +361 -0
  64. data/lib/io_streams/tabular.rb +344 -30
  65. data/lib/io_streams/utils.rb +441 -6
  66. data/lib/io_streams/version.rb +1 -1
  67. data/lib/io_streams/writer.rb +27 -0
  68. data/lib/io_streams/xlsx/cells.rb +127 -0
  69. data/lib/io_streams/xlsx/reader.rb +56 -0
  70. data/lib/io_streams/xlsx/schema_reader.rb +90 -0
  71. data/lib/io_streams/xlsx/v3_writer.rb +16 -0
  72. data/lib/io_streams/zip/reader.rb +7 -0
  73. data/lib/io_streams/zip/writer.rb +24 -2
  74. data/lib/iostreams.rb +21 -17
  75. metadata +28 -2
@@ -5,6 +5,10 @@ module IOStreams
5
5
  class S3 < IOStreams::Path
6
6
  attr_reader :bucket_name, :options
7
7
 
8
+ # The secret access key, also within `client:`, and the customer encryption keys, see
9
+ # IOStreams::Path.redact_options. A session token and credentials are always sensitive by name.
10
+ def self.sensitive_option_names = %i[secret_access_key sse_customer_key copy_source_sse_customer_key]
11
+
8
12
  # Largest file size supported by the S3 copy object api.
9
13
  S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
10
14
 
@@ -178,14 +182,55 @@ module IOStreams
178
182
  @client_options[:region] = region if region
179
183
  end
180
184
 
181
- @options = args
182
- @options.merge!(uri.query.transform_keys(&:to_sym)) if uri.query
185
+ @options = uri.query ? args.merge(uri.query.transform_keys(&:to_sym)) : args
183
186
 
184
187
  super(key)
188
+ # Each child that #each_child yields is a new path too, so nothing is checked while deprecation warnings are off.
189
+ return unless Warning[:deprecated]
190
+
191
+ warn_unusable_url(uri)
192
+ warn_client_arguments(client, access_key_id: access_key_id, secret_access_key: secret_access_key, region: region)
193
+ warn_unusable_options(args.keys) unless args.empty?
194
+ warn_unusable_query(uri.query) if uri.query
195
+ end
196
+
197
+ # The S3 requests whose parameters IOStreams 3.0 takes as the options of a path, see .known_option_names.
198
+ OPTION_OPERATIONS = %i[get_object head_object put_object copy_object delete_object delete_objects list_objects_v2].freeze
199
+ private_constant :OPTION_OPERATIONS
200
+
201
+ # The parameters of the S3 requests that a path sets itself, such as `key`, which it overrides, and those of a
202
+ # listing that would change which keys it returns, or how, which IOStreams 3.0 does not take as the options of a
203
+ # path, see .known_option_names.
204
+ PATH_PARAMETERS = %i[
205
+ bucket key body response_target copy_source prefix continuation_token delimiter delete encoding_type start_after
206
+ ].freeze
207
+ private_constant :PATH_PARAMETERS
208
+
209
+ # Returns [Array<Symbol>] the options of a path that IOStreams 3.0 takes, which raises ArgumentError for any other
210
+ # when the path is created: the parameters of its S3 requests, see OPTION_OPERATIONS, as the AWS SDK names them,
211
+ # other than those that the path sets itself, see PATH_PARAMETERS. Load the AWS SDK first.
212
+ def self.known_option_names
213
+ @known_option_names ||=
214
+ (OPTION_OPERATIONS.flat_map { |operation| ::Aws::S3::Client.api.operation(operation).input.shape.member_names }.uniq -
215
+ PATH_PARAMETERS).freeze
185
216
  end
186
217
 
187
218
  def to_s
188
- ::File.join("s3://", bucket_name, path)
219
+ Utils::PathName.join("s3://", bucket_name, path)
220
+ end
221
+
222
+ # Joining a name that starts with `/` to the bucket itself, see `IOStreams::Path#join`, gives the key that starts
223
+ # with `/`, such as `/exports/a.csv`, while IOStreams v3.0 joins it as a key within the bucket, `exports/a.csv`, as
224
+ # a name joined to the local directory `/` is within it. So it is deprecated, and reports a deprecation warning,
225
+ # see `IOStreams::Deprecation`.
226
+ def join(*elements)
227
+ joined = super
228
+ if path.empty? && joined.path.start_with?("/")
229
+ Deprecation.warn("joining a name that starts with / to an S3 bucket is deprecated, since v3.0 joins it as " \
230
+ "a key within the bucket, such as exports/a.csv for /exports/a.csv: remove the leading /, " \
231
+ "or name a key that starts with / in the url, such as s3://bucket//exports/a.csv")
232
+ end
233
+ joined
189
234
  end
190
235
 
191
236
  # Does not support relative file names since there is no concept of current working directory
@@ -227,9 +272,10 @@ module IOStreams
227
272
  target = IOStreams.new(target_path)
228
273
  return super(target, convert: convert, **args) unless target.is_a?(self.class)
229
274
 
275
+ warn_unconverted_copy_options(args[:mode], args.except(:mode))
230
276
  authorize!
231
277
  target.authorize!
232
- source_name = ::File.join(bucket_name, path)
278
+ source_name = Utils::PathName.join(bucket_name, path)
233
279
  client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
234
280
  target
235
281
  end
@@ -243,9 +289,10 @@ module IOStreams
243
289
  return super(source, convert: convert, **args)
244
290
  end
245
291
 
292
+ warn_unconverted_copy_options(args[:mode], args.except(:mode))
246
293
  authorize!
247
294
  source.authorize!
248
- source_name = ::File.join(source.bucket_name, source.path)
295
+ source_name = Utils::PathName.join(source.bucket_name, source.path)
249
296
  client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
250
297
  end
251
298
 
@@ -339,7 +386,7 @@ module IOStreams
339
386
  each_object(prefix) do |name, object|
340
387
  next if !directories && object.key.end_with?("/")
341
388
 
342
- file_name = ::File.join("s3://", name, object.key)
389
+ file_name = Utils::PathName.join("s3://", name, object.key)
343
390
  next unless matcher.match?(file_name)
344
391
 
345
392
  child = child_path(name, object.key)
@@ -362,6 +409,131 @@ module IOStreams
362
409
 
363
410
  private
364
411
 
412
+ # Reports a deprecation warning for each part of the url that this path cannot use, which IOStreams 3.0 raises
413
+ # ArgumentError for when the path is created, so that a mistake in a configured url is reported wherever the
414
+ # code runs: a url without a bucket, a fragment, which an unencoded `#` starts, and which is not part of the key, a
415
+ # port, and a user name or password, which are ignored.
416
+ def warn_unusable_url(uri)
417
+ warn_url("it names no bucket, name one, such as 's3://bucket/#{path}'") if bucket_name.to_s.empty?
418
+ if uri.fragment
419
+ warn_url("it has a fragment, which a '#' starts, and which is not part of the key, percent-encode a '#' in a " \
420
+ "key as '%23'")
421
+ end
422
+ if uri.port
423
+ warn_url("its port is ignored, supply the endpoint of another service that implements the S3 API with " \
424
+ "client: {endpoint: \"https://host:port\"}")
425
+ end
426
+ return unless uri.user || uri.password
427
+
428
+ warn_url("its user name and password are ignored, supply the credentials with access_key_id: and " \
429
+ "secret_access_key:")
430
+ end
431
+
432
+ # Reports that the S3 url of this path is deprecated, since IOStreams 3.0 raises ArgumentError for it, for the
433
+ # supplied reason.
434
+ def warn_url(reason)
435
+ Deprecation.warn("the S3 url #{warning_url} is deprecated, since #{reason}")
436
+ end
437
+
438
+ # Returns [String] the url of this path as a deprecation warning names it, quoted: its bucket and its key, without
439
+ # the parts of the url that the path cannot use, such as a user name, and with an empty bucket kept, unlike #to_s,
440
+ # such as `"s3:///a.csv"`.
441
+ def warning_url
442
+ Utils.display_text("s3://#{bucket_name}/#{path}").inspect
443
+ end
444
+
445
+ # Reports a deprecation warning for the region and credentials supplied with an `Aws::S3::Client`, which ignores
446
+ # them, since it has its own, and which IOStreams 3.0 raises ArgumentError for when the path is created.
447
+ def warn_client_arguments(client, **arguments)
448
+ return if client.nil? || client.is_a?(Hash)
449
+
450
+ names = arguments.compact.keys
451
+ return if names.empty?
452
+
453
+ words = Deprecation::Words.new(names)
454
+ Deprecation.warn("#{words.keywords} #{words.are} deprecated with an Aws::S3::Client as client:, which ignores " \
455
+ "#{words.them}, since it has its own, supply #{words.them} to Aws::S3::Client.new instead, or " \
456
+ "supply client: as a Hash of its options")
457
+ end
458
+
459
+ # Reports a deprecation warning for the options of this path that IOStreams 3.0 raises ArgumentError for when the
460
+ # path is created, see #unusable_option_names: one that the path sets itself, such as `key:`, which it overrides,
461
+ # and one that no S3 request takes, such as a misspelled one, which the AWS SDK raises for once the path is used.
462
+ def warn_unusable_options(names)
463
+ unusable_option_names(names).each_pair do |set_by_path, unusable|
464
+ words = Deprecation::Words.new(unusable)
465
+ warn_unusable("the S3 #{words.one? ? 'option' : 'options'} #{words.keywords}", words, set_by_path)
466
+ end
467
+ end
468
+
469
+ # Reports a deprecation warning for the names in the query of the url of this path that IOStreams 3.0 raises
470
+ # ArgumentError for when the path is created, as for an option, see #warn_unusable_options. A name without a
471
+ # value is more likely part of a key, such as the `?` of `s3://bucket/data/file?.csv`, so the warning then also
472
+ # says how to write a `?` in a key.
473
+ def warn_unusable_query(query)
474
+ unusable_option_names(query.keys.map(&:to_sym)).each_pair do |set_by_path, unusable|
475
+ names = unusable.map(&:to_s)
476
+ words = Deprecation::Words.new(names)
477
+ encode = ", and percent-encode a '?' in a key as '%3F'" if names.any? { |name| query[name].to_s.empty? }
478
+ warn_unusable("#{words.keywords} in the query of the S3 url #{warning_url}", words, set_by_path, encode)
479
+ end
480
+ end
481
+
482
+ # Returns [Hash<true|false, Array>] the supplied names of options that IOStreams 3.0 raises ArgumentError for: by
483
+ # whether the path sets them itself (true), see PATH_PARAMETERS, or no S3 request of a path takes them (false),
484
+ # see .known_option_names, without either when there are none.
485
+ def unusable_option_names(names)
486
+ set_by_path = names & PATH_PARAMETERS
487
+ {true => set_by_path, false => names - set_by_path - self.class.known_option_names}.reject { |_, list| list.empty? }
488
+ end
489
+
490
+ # Reports that the options, which the subject names, and the words are about, see `IOStreams::Deprecation::Words`,
491
+ # are deprecated, since the path sets them itself, or since no S3 request takes them, followed by the supplied
492
+ # advice.
493
+ def warn_unusable(subject, words, set_by_path, advice = nil)
494
+ them = words.them
495
+ reason =
496
+ if set_by_path
497
+ "the path sets #{them} itself, and overrides #{them}, so remove #{them}"
498
+ else
499
+ "no S3 request takes #{them}, so correct or remove #{them}"
500
+ end
501
+ Deprecation.warn("#{subject} #{words.are} deprecated, since #{reason}#{advice}")
502
+ end
503
+
504
+ # Returns [Array(Array<Symbol>, String)|nil] the methods of the IO that v3.0 reads the object from, in place of the
505
+ # temp File that v2.2 downloads it into, and its name, see `IOStreams::Stream#v3_source`: v3.0 reads the object as
506
+ # the application reads it, with an IO that only reads, unless the `range` or `part_number` option chooses the
507
+ # bytes to read, which it still downloads into a temp file.
508
+ def v3_source
509
+ return if options.key?(:range) || options.key?(:part_number)
510
+
511
+ [IOStreams::Reader::V3_READ_METHODS, "the IO of an S3 path"]
512
+ end
513
+
514
+ # The methods of the IO that v3.0 uploads the object with, see #v3_target: those of an IO that write, and #flush.
515
+ V3_UPLOAD_METHODS = [*IOStreams::Writer::V3_WRITE_METHODS, :flush].freeze
516
+ private_constant :V3_UPLOAD_METHODS
517
+
518
+ # The options that describe the whole object, with which v3.0 still writes the object into a temp file first, as
519
+ # does a checksum of its data, such as `checksum_crc32`, see #v3_target.
520
+ V3_WHOLE_OBJECT_OPTIONS = %i[content_length content_md5 write_offset_bytes].freeze
521
+ private_constant :V3_WHOLE_OBJECT_OPTIONS
522
+
523
+ # Returns [Array(Array<Symbol>, String)|nil] the methods of the IO that v3.0 uploads the object with, in place of the
524
+ # temp File that v2.2 writes it into, and its name, see `IOStreams::Stream#v3_target`: v3.0 uploads the object as
525
+ # the application writes it, with an IO that only writes, unless an option describes the whole object, which it
526
+ # still writes into a temp file first. v3.0 writes the object with only the options that `put_object` takes, so
527
+ # only those decide, rather than one that only another request takes, such as `checksum_mode` when reading.
528
+ def v3_target
529
+ put_object = ::Aws::S3::Client.api.operation(:put_object).input.shape
530
+ whole_object = options.each_key.any? do |name|
531
+ put_object.member?(name) &&
532
+ (V3_WHOLE_OBJECT_OPTIONS.include?(name) || (name.start_with?("checksum_") && name != :checksum_algorithm))
533
+ end
534
+ [V3_UPLOAD_METHODS, "the IO of an S3 path"] unless whole_object
535
+ end
536
+
365
537
  # Yields the bucket name and each object in the bucket whose key starts with the supplied prefix.
366
538
  def each_object(prefix)
367
539
  token = nil
@@ -391,9 +563,8 @@ module IOStreams
391
563
  #
392
564
  # The child uses this path's client, so that it has the same credentials and region.
393
565
  def child_path(bucket_name, key)
394
- child = self.class.new("s3://#{bucket_name}", client: client)
395
- child.path = key.dup.freeze
396
- child
566
+ child = self.class.new("s3://#{bucket_name}", client: client)
567
+ child.replace_path(key.dup.freeze)
397
568
  end
398
569
  end
399
570
  end
@@ -36,6 +36,20 @@ module IOStreams
36
36
 
37
37
  attr_reader :hostname, :username, :ssh_options, :url, :port
38
38
 
39
+ # The password, and the private key supplied as the ssh option `IdentityKey`, see IOStreams::Path.redact_options.
40
+ def self.sensitive_option_names = %i[password identity_key]
41
+
42
+ # Returns [String|nil] the deprecation warning for an SFTP url without a path, such as `sftp://host`, which
43
+ # IOStreams v3.0 reads as `sftp://host/`, the root directory, see `IOStreams::Path.name_deprecation`, while this
44
+ # version lists the login directory for it, see #each_child.
45
+ def self.name_deprecation(url)
46
+ return unless url_without_path?(url)
47
+
48
+ "an SFTP url without a path, such as sftp://host, is deprecated, since v3.0 reads it as sftp://host/, the " \
49
+ "root directory, which changes its #to_s and #path, and what #each_child lists: supply the url with the /, " \
50
+ "which v2.2 reads the same way"
51
+ end
52
+
39
53
  # Stream to a remote file over sftp.
40
54
  #
41
55
  # url: [String]
@@ -44,7 +58,8 @@ module IOStreams
44
58
  # SECURITY WARNING:
45
59
  # A username and password supplied in the url remain part of it, so `#to_s` and `#url`
46
60
  # return them, as does any log or error message that includes the path.
47
- # Supply them with the `username:` and `password:` arguments instead.
61
+ # Supply them with the `username:` and `password:` arguments instead,
62
+ # and log `#display_name`, which never includes them.
48
63
  #
49
64
  # username: [String]
50
65
  # Name of user to login with.
@@ -117,6 +132,7 @@ module IOStreams
117
132
  validate_username!
118
133
 
119
134
  super(uri.path)
135
+ warn_unusable_url(uri)
120
136
  end
121
137
 
122
138
  # Does not support relative file names since there is no concept of current working directory
@@ -128,6 +144,11 @@ module IOStreams
128
144
  url
129
145
  end
130
146
 
147
+ # Returns [String] the url without the user name, password, query or fragment, which can hold credentials.
148
+ def display_name
149
+ url.sub(%r{\A([^:/]+://)[^/?#]*@}, "\\1").sub(/[?#].*\z/m, "")
150
+ end
151
+
131
152
  # Note that mkdir is delayed and only executed when the file write is performed.
132
153
  def mkdir
133
154
  @mkdir = true
@@ -194,6 +215,29 @@ module IOStreams
194
215
  "sftp://#{hostname.to_s.downcase}:#{port}#{normalize_path(path)}".chomp("/")
195
216
  end
196
217
 
218
+ # Reports a deprecation warning for each part of the url that this path cannot use, which IOStreams 3.0 raises
219
+ # ArgumentError for when the path is created, so that a mistake in a configured url is reported wherever the
220
+ # code runs: a url without a host, a query, whose ssh options are ignored, and a fragment, which an unencoded `#`
221
+ # starts, and which is not part of the name. Each warning names the url without its user name, password, query or
222
+ # fragment, see #display_name.
223
+ def warn_unusable_url(uri)
224
+ warn_url("it names no host, name one, such as 'sftp://example.org#{path}'") if uri.hostname.to_s.empty?
225
+ if uri.query
226
+ warn_url("its query is ignored, supply ssh options with ssh_options:, and percent-encode a '?' in a name as " \
227
+ "'%3F'")
228
+ end
229
+ return unless uri.fragment
230
+
231
+ warn_url("it has a fragment, which a '#' starts, and which is not part of the name, percent-encode a '#' in a " \
232
+ "name as '%23'")
233
+ end
234
+
235
+ # Reports that the url of this path is deprecated, since IOStreams 3.0 raises ArgumentError for it, for the
236
+ # supplied reason.
237
+ def warn_url(reason)
238
+ Deprecation.warn("the SFTP url #{display_name.inspect} is deprecated, since #{reason}")
239
+ end
240
+
197
241
  # Usernames are passed to the `sftp` executable, so reject values that it could treat as options.
198
242
  def validate_username!
199
243
  return if username.nil?
@@ -209,8 +253,8 @@ module IOStreams
209
253
  def child_path(name)
210
254
  server = port == 22 ? "sftp://#{hostname}" : "sftp://#{hostname}:#{port}"
211
255
  child = self.class.new(server, username: username, password: password, ssh_options: ssh_options)
212
- child.path = (path.empty? ? "/#{name}" : ::File.join(path, name)).freeze
213
- child.url = "#{server}#{child.path}"
256
+ child.replace_path((path.empty? ? "/#{name}" : Utils::PathName.join(path, name)).freeze)
257
+ child.url = "#{server}#{child.path}"
214
258
  child
215
259
  end
216
260
 
@@ -0,0 +1,23 @@
1
+ module IOStreams
2
+ # The path classes of the schemes of urls, such as `IOStreams::Paths::S3` for `s3://`, see `IOStreams.register_scheme`.
3
+ module Paths
4
+ # Returns [IOStreams::Path] the path of the supplied name or url, created with the supplied arguments by the path
5
+ # class of its scheme, see .path_class.
6
+ #
7
+ # It reports no deprecation warning for the name, which `IOStreams.path` reports for a name that the application
8
+ # supplies, see `IOStreams::Path.name_deprecation`, so that IOStreams can create the path of a name that it derives
9
+ # from one, such as the directory that a pattern is matched within, see `IOStreams::Paths::Matcher`.
10
+ def self.build(name, **args)
11
+ klass = path_class(name)
12
+ args.empty? ? klass.new(name) : klass.new(name, **args)
13
+ end
14
+
15
+ # Returns [Class] the path class of the scheme of the supplied name or url, such as `IOStreams::Paths::SFTP`, or the
16
+ # one registered for a name without a scheme, such as `IOStreams::Paths::File`, see `IOStreams.scheme`.
17
+ #
18
+ # Raises ArgumentError when the scheme is not registered, and `URI::Error` when the url is not valid.
19
+ def self.path_class(name)
20
+ IOStreams.scheme(name.include?("://") ? Utils::URI.new(name).scheme : nil)
21
+ end
22
+ end
23
+ end
@@ -7,6 +7,24 @@ module IOStreams
7
7
  %i[passphrase ignore_mdc_error verify_first]
8
8
  end
9
9
 
10
+ def self.sensitive_option_names
11
+ %i[passphrase]
12
+ end
13
+
14
+ # The methods that the stream that v3.0 supplies to a reader block for PGP answers, when gpg does not read a regular
15
+ # local file, see .v3_stream_methods: those of an IO that read, and #binmode and #binmode?.
16
+ V3_METHODS = [*V3_READ_METHODS, :binmode, :binmode?].freeze
17
+ private_constant :V3_METHODS
18
+
19
+ # Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a reader block, in place of the IO
20
+ # of gpg's output that this reader yields, answers, see `IOStreams::Reader.v3_stream_methods`: nil with
21
+ # `verify_first: true`, since both versions then yield the File that gpg decrypts the data into, or when it reads
22
+ # a regular local file, which gpg reads itself in v3.0, so that the block receives gpg's output in both versions,
23
+ # otherwise those of the stream that v3.0 supplies, which feeds gpg the data that it reads.
24
+ def self.v3_stream_methods(options)
25
+ V3_METHODS unless options[:verify_first] || yield
26
+ end
27
+
10
28
  # Passphrase to use to open the private key to decrypt the received file
11
29
  class << self
12
30
  attr_writer :default_passphrase
@@ -11,6 +11,22 @@ module IOStreams
11
11
  compress compress_level]
12
12
  end
13
13
 
14
+ def self.sensitive_option_names
15
+ %i[signer_passphrase]
16
+ end
17
+
18
+ # The methods that the stream that v3.0 supplies to a writer block for PGP answers: those of an IO that write,
19
+ # #flush, #binmode and #binmode?, see .v3_stream_methods.
20
+ V3_METHODS = [*V3_WRITE_METHODS, :flush, :binmode, :binmode?].freeze
21
+ private_constant :V3_METHODS
22
+
23
+ # Returns [Array<Symbol>] the methods that the stream that v3.0 supplies to a writer block, in place of the IO of
24
+ # gpg's input that this writer yields, answers, which writes to gpg, whatever it writes to, see
25
+ # `IOStreams::Writer.v3_stream_methods`.
26
+ def self.v3_stream_methods(_options)
27
+ V3_METHODS
28
+ end
29
+
14
30
  class << self
15
31
  # Sign all encrypted files with this users key.
16
32
  # Default: Do not sign encrypted files.
@@ -262,8 +262,9 @@ module IOStreams
262
262
  # Returns [String] containing all the public keys for the supplied email address or key id.
263
263
  #
264
264
  # Raises ArgumentError when neither :email nor :key_id is supplied.
265
- # Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added.
266
- # In v3.0 this will raise ArgumentError instead.
265
+ # Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added, which is
266
+ # deprecated, since v3.0 raises ArgumentError for it instead: it reports a deprecation warning, see
267
+ # `IOStreams::Deprecation`.
267
268
  #
268
269
  # email: [String] Email address for requested key.
269
270
  #
@@ -276,6 +277,9 @@ module IOStreams
276
277
  if email.nil? && key_id.nil?
277
278
  raise(ArgumentError, "Either :email, or :key_id must be supplied") if email_omitted
278
279
 
280
+ Deprecation.warn("IOStreams::Pgp.export with email: nil and without a key_id: is deprecated, since v3.0 " \
281
+ "raises ArgumentError for it rather than IOStreams::Pgp::Failure: supply the email: or " \
282
+ "key_id: of the key")
279
283
  raise(Pgp::Failure, "GPG Failed reading key: no email address or key id was supplied")
280
284
  end
281
285
 
@@ -490,6 +494,10 @@ module IOStreams
490
494
  #
491
495
  # After importing keys, they are not trusted and the relevant trust level must be set.
492
496
  #
497
+ # Without an :email or :key_id, it raises Pgp::Failure, since gpg rejects the empty user id that it is asked for,
498
+ # which is deprecated, since v3.0 raises ArgumentError instead: it reports a deprecation warning, see
499
+ # `IOStreams::Deprecation`.
500
+ #
493
501
  # key_id: [String]
494
502
  # The fingerprint of the key, as hexadecimal digits only.
495
503
  # Raises ArgumentError when it contains any other characters.
@@ -511,6 +519,12 @@ module IOStreams
511
519
  # key at this level allows that attacker to impersonate other recipients.
512
520
  # When the key cannot be fully verified, supply a lower `level`.
513
521
  def self.set_trust(email: nil, key_id: nil, level: 5)
522
+ if email.nil? && key_id.nil?
523
+ Deprecation.warn("IOStreams::Pgp.set_trust without an email: or key_id: is deprecated, since v3.0 raises " \
524
+ "ArgumentError for it rather than IOStreams::Pgp::Failure: supply the email: or key_id: of " \
525
+ "the key")
526
+ end
527
+
514
528
  # The key_id is written into gpg's ownertrust input, where any other character, such as a newline,
515
529
  # could add trust lines for other keys.
516
530
  if key_id && !key_id.to_s.match?(/\A\h+\z/)
@@ -1,5 +1,8 @@
1
1
  module IOStreams
2
2
  class Reader
3
+ # Answers the names of the options that this reader takes in IOStreams 3.0, and those that it replaces.
4
+ extend Deprecation::OptionNames
5
+
3
6
  # Returns [Array<Symbol>] the names of the options this reader accepts,
4
7
  # or [nil] when the reader does not declare them.
5
8
  #
@@ -9,6 +12,47 @@ module IOStreams
9
12
  nil
10
13
  end
11
14
 
15
+ # Returns [Array<Symbol>] the names of the options whose values must not be displayed, such as a passphrase,
16
+ # so that `#inspect` on a path does not display them, see `IOStreams::Extension#redact_options`.
17
+ def self.sensitive_option_names
18
+ []
19
+ end
20
+
21
+ # Returns [Symbol|nil] the tabular format of the text that this reader turns its data into, such as `:csv` for a
22
+ # spreadsheet, otherwise nil, for a reader whose data is the text, such as gzip, see
23
+ # `IOStreams::Builder#text_format`.
24
+ def self.text_format
25
+ nil
26
+ end
27
+
28
+ # The methods of an IO that read, which every stream that v3.0 supplies to a reader block answers, see
29
+ # .v3_stream_methods.
30
+ V3_READ_METHODS = %i[read readpartial gets readline readlines each_line each eof? eof closed?].freeze
31
+
32
+ # Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a reader block, in place of the one
33
+ # that this reader yields, answers, when it does not answer all of the methods of that one, such as when it is the
34
+ # reader of a gem, otherwise nil, for a reader that yields the same kind of stream in both versions, see
35
+ # `IOStreams::Builder#v3_stream`.
36
+ #
37
+ # Parameters:
38
+ # options: [Hash]
39
+ # The options that this reader is opened with, such as `verify_first: true` for PGP, see
40
+ # `IOStreams::Pgp::Reader.v3_stream_methods`.
41
+ #
42
+ # Yields, for a reader whose stream depends on it, to ask whether this reader reads a regular local file in v3.0,
43
+ # rather than another stream, an IO that is not a file, or the object of an S3 or HTTP path, which v3.0 reads without
44
+ # a temp file. The block answers true or false.
45
+ def self.v3_stream_methods(_options)
46
+ nil
47
+ end
48
+
49
+ # Returns [Hash<Symbol:String>|nil] what the application uses in place of each method of the stream that this reader
50
+ # yields that v3.0 removes, when it is not the methods that the stream of v3.0 answers, see .v3_stream_methods, such as
51
+ # a setting of the path, otherwise nil, see `IOStreams::Builder#v3_stream`.
52
+ def self.v3_stream_advice
53
+ nil
54
+ end
55
+
12
56
  # When a Reader does not support streams, we copy the stream to a local temp file
13
57
  # and then pass that filename in for this reader.
14
58
  def self.stream(input_stream, **args, &block)