iostreams 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/docs/config.md +13 -9
  3. data/docs/copy_files.md +20 -2
  4. data/docs/extensions.md +18 -4
  5. data/docs/formats.md +644 -16
  6. data/docs/path.md +18 -2
  7. data/docs/streams.md +54 -9
  8. data/docs/tutorial.md +14 -2
  9. data/docs/upgrading.md +470 -23
  10. data/lib/io_streams/builder/deprecations.rb +250 -0
  11. data/lib/io_streams/builder.rb +256 -39
  12. data/lib/io_streams/bzip2/reader.rb +18 -2
  13. data/lib/io_streams/bzip2/writer.rb +11 -2
  14. data/lib/io_streams/deprecation/methods.rb +51 -0
  15. data/lib/io_streams/deprecation/once.rb +27 -0
  16. data/lib/io_streams/deprecation/option_names.rb +26 -0
  17. data/lib/io_streams/deprecation/words.rb +42 -0
  18. data/lib/io_streams/deprecation.rb +259 -0
  19. data/lib/io_streams/encode/reader.rb +54 -3
  20. data/lib/io_streams/encode/writer.rb +19 -2
  21. data/lib/io_streams/encode.rb +99 -0
  22. data/lib/io_streams/errors.rb +51 -0
  23. data/lib/io_streams/gzip/reader.rb +11 -0
  24. data/lib/io_streams/io_streams.rb +365 -35
  25. data/lib/io_streams/line/reader.rb +2 -1
  26. data/lib/io_streams/line/utf8_reader.rb +23 -0
  27. data/lib/io_streams/path.rb +125 -35
  28. data/lib/io_streams/paths/file.rb +72 -9
  29. data/lib/io_streams/paths/http.rb +48 -2
  30. data/lib/io_streams/paths/matcher.rb +6 -3
  31. data/lib/io_streams/paths/s3.rb +180 -9
  32. data/lib/io_streams/paths/sftp.rb +47 -3
  33. data/lib/io_streams/paths.rb +23 -0
  34. data/lib/io_streams/pgp/reader.rb +18 -0
  35. data/lib/io_streams/pgp/writer.rb +16 -0
  36. data/lib/io_streams/pgp.rb +16 -2
  37. data/lib/io_streams/reader.rb +44 -0
  38. data/lib/io_streams/record/reader.rb +59 -68
  39. data/lib/io_streams/record/writer.rb +17 -10
  40. data/lib/io_streams/row/header_row.rb +21 -0
  41. data/lib/io_streams/row/reader.rb +52 -15
  42. data/lib/io_streams/row/writer.rb +64 -13
  43. data/lib/io_streams/schema/columns.rb +456 -0
  44. data/lib/io_streams/schema/constraints.rb +99 -0
  45. data/lib/io_streams/schema/field.rb +967 -0
  46. data/lib/io_streams/schema/json_number.rb +56 -0
  47. data/lib/io_streams/schema/keys.rb +78 -0
  48. data/lib/io_streams/schema/pattern.rb +257 -0
  49. data/lib/io_streams/schema/source.rb +31 -0
  50. data/lib/io_streams/schema.rb +616 -0
  51. data/lib/io_streams/stream.rb +836 -67
  52. data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
  53. data/lib/io_streams/tabular/header.rb +29 -0
  54. data/lib/io_streams/tabular/header_mapping.rb +332 -0
  55. data/lib/io_streams/tabular/mapping.rb +165 -0
  56. data/lib/io_streams/tabular/parser/array.rb +5 -0
  57. data/lib/io_streams/tabular/parser/base.rb +24 -0
  58. data/lib/io_streams/tabular/parser/fixed.rb +197 -0
  59. data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
  60. data/lib/io_streams/tabular/parser/hash.rb +10 -0
  61. data/lib/io_streams/tabular/parser/json.rb +17 -0
  62. data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
  63. data/lib/io_streams/tabular/schema_mapping.rb +361 -0
  64. data/lib/io_streams/tabular.rb +344 -30
  65. data/lib/io_streams/utils.rb +441 -6
  66. data/lib/io_streams/version.rb +1 -1
  67. data/lib/io_streams/writer.rb +27 -0
  68. data/lib/io_streams/xlsx/cells.rb +127 -0
  69. data/lib/io_streams/xlsx/reader.rb +56 -0
  70. data/lib/io_streams/xlsx/schema_reader.rb +90 -0
  71. data/lib/io_streams/xlsx/v3_writer.rb +16 -0
  72. data/lib/io_streams/zip/reader.rb +7 -0
  73. data/lib/io_streams/zip/writer.rb +24 -2
  74. data/lib/iostreams.rb +21 -17
  75. metadata +28 -2
@@ -35,41 +35,134 @@ module IOStreams
35
35
  # # => "files/file.xls"
36
36
  #
37
37
  # For Files
38
- # IOStreams.path('blah.zip').option(:encode, encoding: 'BINARY').each(:line) { |line| puts line }
39
- # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:line) { |line| puts line }
40
- # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').each(:hash) { |hash| p hash }
41
- # IOStreams.path('blah.zip').option(:encode, encoding: 'UTF-8').read
38
+ # IOStreams.path('blah.zip').encoding('BINARY').each(:line) { |line| puts line }
39
+ # IOStreams.path('blah.zip').encoding('UTF-8').each(:line) { |line| puts line }
40
+ # IOStreams.path('blah.zip').encoding('UTF-8').each(:hash) { |hash| p hash }
41
+ # IOStreams.path('blah.zip').encoding('UTF-8').read
42
42
  # IOStreams.path('blah.csv.zip').each(:line) { |line| puts line }
43
43
  # IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').read
44
44
  # IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
45
- # IOStreams.path('blah.zip').stream(:zip).stream(:encode, encoding: 'BINARY').read
45
+ # IOStreams.path('blah.zip').stream(:zip).encoding('BINARY').read
46
46
  #
47
+ # A name that IOStreams v3.0 reads as another file or url, such as a local name starting with `~/`, a `file://` url, or
48
+ # an SFTP or HTTP url without a path, is deprecated, and reports a deprecation warning, see
49
+ # `IOStreams::Path.name_deprecation`.
47
50
  def self.path(*elements, **args)
51
+ build_path(elements, args, :name_deprecation)
52
+ end
53
+
54
+ # Returns [Hash] the options for creating the path of the url with `IOStreams.path(url, **options)`, with the value
55
+ # of each sensitive option, such as a password, replaced with "[FILTERED]", so that they can be displayed, for
56
+ # example by an application that stores a url and its options to create the path later.
57
+ #
58
+ # The path class of the url's scheme decides which options are sensitive, see `IOStreams::Path.redact_options`.
59
+ # Every value is replaced when the url is not valid, since then that class is not known, or when the class
60
+ # registered for the scheme, see `IOStreams.register_scheme`, cannot say which are sensitive.
61
+ #
62
+ # Example:
63
+ # IOStreams.redact_path_options("sftp://example.org/a.csv", username: "jack", password: "secret")
64
+ # # => {username: "jack", password: "[FILTERED]"}
65
+ def self.redact_path_options(url, options)
66
+ klass = Paths.path_class(url.to_s)
67
+ return klass.redact_options(options) if klass.respond_to?(:redact_options)
68
+
69
+ options.to_h { |name, _value| [name, Utils::FILTERED] }
70
+ rescue ArgumentError, ::URI::Error
71
+ options.to_h { |name, _value| [name, Utils::FILTERED] }
72
+ end
73
+
74
+ # Returns [Hash<Symbol:Hash>] the options of each stream, as supplied to `IOStreams::Path#stream`, with the value of
75
+ # each sensitive option, such as a PGP passphrase, replaced with "[FILTERED]", so that they can be displayed.
76
+ #
77
+ # The extension registered for each stream decides which of its options are sensitive, see
78
+ # `IOStreams::Extension#redact_options`, as it does for `#inspect`. Every value of a stream that is not registered is
79
+ # replaced, as are options that are not a Hash.
80
+ #
81
+ # Example:
82
+ # IOStreams.redact_stream_options(pgp: {passphrase: "secret", recipient: "a@b.org"})
83
+ # # => {pgp: {passphrase: "[FILTERED]", recipient: "a@b.org"}}
84
+ def self.redact_stream_options(streams)
85
+ Utils.redact_streams(streams) { |name| @extensions[name.to_s.to_sym] }
86
+ end
87
+
88
+ # Returns [IOStreams::Path] the path of the supplied elements, joined into one name or url, created with the supplied
89
+ # arguments by the path class of its scheme, see `IOStreams::Paths.build`, reporting the deprecation warning, if any,
90
+ # that the supplied class method of that class gives for the name, see `IOStreams::Path.name_deprecation`, as the
91
+ # application supplied it.
92
+ def self.build_path(elements, args, deprecation)
48
93
  return elements.first if (elements.size == 1) && args.empty? && elements.first.is_a?(IOStreams::Path)
49
94
 
50
- elements = elements.collect(&:to_s)
51
- path = ::File.join(*elements)
52
- extracted_scheme = path.include?("://") ? Utils::URI.new(path).scheme : nil
53
- klass = scheme(extracted_scheme)
54
- args.empty? ? klass.new(path) : klass.new(path, **args)
95
+ name = Utils::PathName.join(*elements.collect(&:to_s))
96
+ path = Paths.build(name, **args)
97
+ warn_deprecated_name(path.class, deprecation, name)
98
+ path
99
+ end
100
+ private_class_method :build_path
101
+
102
+ # Reports the deprecation warning that the supplied class method of the path class gives for a name or url that the
103
+ # application supplied, if any: `:name_deprecation` when IOStreams v3.0 reads it as another file or url, or
104
+ # `:location_deprecation` when it reads it as another location, which the allowed paths compare, see
105
+ # `IOStreams::Path.name_deprecation`.
106
+ def self.warn_deprecated_name(klass, deprecation, name)
107
+ message = klass.public_send(deprecation, name) if klass.respond_to?(deprecation)
108
+ Deprecation.warn(message) if message
109
+ end
110
+ private_class_method :warn_deprecated_name
111
+
112
+ # Reports a deprecation warning for the name of the directory of a pattern, see .warn_deprecated_name. A pattern that
113
+ # is not a valid url, such as one with braces, is matched within the directory before its first pattern character,
114
+ # which the warnings do not apply to, see `IOStreams::Paths::Matcher`.
115
+ def self.warn_deprecated_pattern(pattern)
116
+ klass = Paths.path_class(pattern)
117
+ rescue ArgumentError, ::URI::Error
118
+ nil
119
+ else
120
+ warn_deprecated_name(klass, :name_deprecation, pattern)
121
+ end
122
+ private_class_method :warn_deprecated_pattern
123
+
124
+ # Returns [true|false] whether the object is a `Pathname`, which IOStreams v3.0 reads as the name of a file.
125
+ def self.pathname?(object)
126
+ return false unless defined?(::Pathname)
127
+
128
+ object.is_a?(::Pathname)
55
129
  end
130
+ private_class_method :pathname?
56
131
 
57
132
  # For an existing IO Stream
58
133
  # IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').read
59
134
  # IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').each(:line){ ... }
60
135
  # IOStreams.stream(io).file_name('blah.csv.zip').each(:line) { ... }
61
136
  # IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
137
+ #
138
+ # A `Pathname` is deprecated, since IOStreams v3.0 raises ArgumentError for it, as for a String: it reports a
139
+ # deprecation warning, see `IOStreams::Deprecation`.
62
140
  def self.stream(io_stream)
63
141
  return io_stream if io_stream.is_a?(Stream)
64
142
 
143
+ if pathname?(io_stream)
144
+ Deprecation.warn("a Pathname supplied to IOStreams.stream is deprecated, since v3.0 raises ArgumentError for it, " \
145
+ "as for a String: supply IOStreams.path(pathname) instead, and call #raw on it to read or " \
146
+ "write the data as it is stored")
147
+ end
65
148
  Stream.new(io_stream)
66
149
  end
67
150
 
68
151
  # For processing by either a file name or an open IO stream.
152
+ #
153
+ # A `Pathname` is deprecated, which it reads and writes as an IO, since IOStreams v3.0 returns the path of the file
154
+ # that it names, which applies the streams of its file name: it reports a deprecation warning, see
155
+ # `IOStreams::Deprecation`. So is one supplied to `#copy_from`, `#copy_to` or `#move_to`, which call it.
69
156
  def self.new(file_name_or_io)
70
157
  return file_name_or_io if file_name_or_io.is_a?(Stream)
71
-
72
- file_name_or_io.is_a?(String) ? path(file_name_or_io) : stream(file_name_or_io)
158
+ return path(file_name_or_io) if file_name_or_io.is_a?(String)
159
+ return stream(file_name_or_io) unless pathname?(file_name_or_io)
160
+
161
+ Deprecation.warn("a Pathname supplied to IOStreams.new, #copy_from, #copy_to or #move_to is deprecated, since " \
162
+ "v3.0 takes it as the name of a file, applying the streams that its file name implies, rather " \
163
+ "than as an IO: supply pathname.to_s instead, and call #raw on its path to read or write the " \
164
+ "data as it is stored")
165
+ Stream.new(file_name_or_io)
73
166
  end
74
167
 
75
168
  # Join the supplied path elements to a root path.
@@ -121,9 +214,14 @@ module IOStreams
121
214
  # IOStreams.temp_file("export", ".csv") { |path| path.write("Hello World") }
122
215
  #
123
216
  # Note: The temp file is accessible even when it is not within the allowed paths, see `IOStreams.add_allowed_path`.
217
+ #
218
+ # The path reads and writes its data as it is stored, whatever its extension, while IOStreams v3.0 applies the streams
219
+ # that its extension implies, such as `.gz`, so reading or writing it without #raw, which reads and writes the data
220
+ # as it is stored on both, is deprecated when its extension implies streams, and reports a deprecation warning, see
221
+ # `IOStreams::Deprecation`.
124
222
  def self.temp_file(basename, extension = "")
125
223
  Utils.temp_file_name(basename, extension) do |file_name|
126
- yield(Paths::File.new(file_name).send(:permit!).stream(:none))
224
+ yield(Paths::File.new(file_name).send(:permit!).send(:no_streams, temp_file: true))
127
225
  end
128
226
  end
129
227
 
@@ -219,6 +317,7 @@ module IOStreams
219
317
  # "**.rb" "lib/song.rb" true
220
318
  # "*" "dave/.profile" true
221
319
  def self.each_child(pattern, case_sensitive: false, directories: false, hidden: false, &block)
320
+ warn_deprecated_pattern(pattern)
222
321
  matcher = Paths::Matcher.new(nil, pattern, case_sensitive: case_sensitive, hidden: hidden)
223
322
 
224
323
  # When the pattern includes an exact file name without any pattern characters
@@ -235,10 +334,20 @@ module IOStreams
235
334
  end
236
335
 
237
336
  # Add a named root path
337
+ #
338
+ # Returns [IOStreams::Path] the root path, which the whole process shares, see `IOStreams.join`. Changing it in place,
339
+ # such as with `#stream` or `#option`, whether through this path, a path supplied as the root, `IOStreams.root`,
340
+ # `IOStreams.roots`, or `IOStreams.join` without elements, is deprecated, and reports a deprecation warning, since v3.0
341
+ # keeps its own frozen copy of the root, which this method returns, and the others return copies of it, so that
342
+ # changing them no longer changes the root: change a copy of it instead, such as `IOStreams.root(:uploads).dup`, or a
343
+ # path joined to it, such as `IOStreams.join("a.csv", root: :uploads)`.
238
344
  def self.add_root(root, *elements, **args)
239
345
  raise(ArgumentError, "Invalid characters in root name #{root.inspect}") unless root.to_s =~ /\A\w+\Z/
240
346
 
241
- @root_paths[root.to_sym] = path(*elements, **args)
347
+ path = path(*elements, **args)
348
+ # The path class of a scheme registered with `IOStreams.register_scheme` need not be an `IOStreams::Path`.
349
+ path.send(:root!, root.to_sym) if path.is_a?(Path)
350
+ @root_paths[root.to_sym] = path
242
351
  end
243
352
 
244
353
  def self.roots
@@ -280,7 +389,7 @@ module IOStreams
280
389
  # * A local file could be replaced with a symbolic link after it is checked but before it is opened.
281
390
  # Allowed paths do not prevent this, so do not allow paths where untrusted users can create files.
282
391
  def self.add_allowed_path(*elements, **args)
283
- location = allowed_location(path(*elements, **args))
392
+ location = allowed_location(build_path(elements, args, :location_deprecation))
284
393
  @allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths + [location]).uniq.freeze }
285
394
  location
286
395
  end
@@ -289,7 +398,7 @@ module IOStreams
289
398
  #
290
399
  # Returns [String] the normalized path that was removed.
291
400
  def self.delete_allowed_path(*elements, **args)
292
- location = allowed_location(path(*elements, **args))
401
+ location = allowed_location(build_path(elements, args, :location_deprecation))
293
402
  @allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths - [location]).freeze }
294
403
  location
295
404
  end
@@ -305,7 +414,7 @@ module IOStreams
305
414
  #
306
415
  # Parameters: Same as `IOStreams.path`
307
416
  def self.allowed_path?(*elements, **args)
308
- path(*elements, **args).send(:allowed?)
417
+ build_path(elements, args, :location_deprecation).send(:allowed?)
309
418
  end
310
419
 
311
420
  def self.allowed_location(path)
@@ -348,22 +457,39 @@ module IOStreams
348
457
  # Since the format is usually inferred from the file name, set this to true when the allowed columns
349
458
  # restrict what an uploaded file can set, so that renaming the file to `.json` cannot bypass them.
350
459
  #
351
- # Default: false. It will default to true in v3.0.
460
+ # Default: false.
352
461
  #
353
462
  # Example:
354
463
  # IOStreams.enforce_column_restrictions = true
464
+ #
465
+ # Deprecated: it only applies to the column options, which are deprecated, see `IOStreams::Stream#each`, and which
466
+ # v3.0 removes, with this setting. Describe the columns of the file with `IOStreams::Stream#schema` instead, whose
467
+ # `fieldsMatch` and `constraints` apply to every input. It still applies to the column options, and reports a
468
+ # deprecation warning, see `IOStreams::Deprecation`, so keep it until they are replaced with a schema. The setting
469
+ # is held by `IOStreams::Tabular::HeaderMapping`, which applies it, and reads it without a deprecation warning.
355
470
  def self.enforce_column_restrictions=(enforce)
356
471
  raise(ArgumentError, "enforce_column_restrictions must be true or false") unless [true, false].include?(enforce)
357
472
 
358
- @enforce_column_restrictions = enforce
473
+ Deprecation.warn("IOStreams.enforce_column_restrictions= #{ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION}")
474
+ Tabular::HeaderMapping.enforce_column_restrictions = enforce
359
475
  end
360
476
 
361
477
  # Returns [true|false] whether column restrictions apply to every input, see `IOStreams.enforce_column_restrictions=`.
478
+ #
479
+ # Deprecated, as `IOStreams.enforce_column_restrictions=` is. It reports a deprecation warning, see
480
+ # `IOStreams::Deprecation`.
362
481
  def self.enforce_column_restrictions?
363
- @enforce_column_restrictions
482
+ Deprecation.warn("IOStreams.enforce_column_restrictions? #{ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION}")
483
+ Tabular::HeaderMapping.enforce_column_restrictions?
364
484
  end
365
485
 
366
- @enforce_column_restrictions = false
486
+ # Why `IOStreams.enforce_column_restrictions=` and `IOStreams.enforce_column_restrictions?` are deprecated, and what
487
+ # to use instead.
488
+ ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION =
489
+ "is deprecated, since it only applies to the column options, such as allowed_columns:, which are deprecated: " \
490
+ "describe the columns of the file with #schema instead, whose fieldsMatch and constraints apply to every input, " \
491
+ "and then remove it".freeze
492
+ private_constant :ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION
367
493
 
368
494
  # Returns [Logger] the logger used by IOStreams for debug logging.
369
495
  #
@@ -384,32 +510,110 @@ module IOStreams
384
510
 
385
511
  # Register a file extension and the reader and writer streaming classes
386
512
  #
513
+ # Returns [IOStreams::Extension] the extension registered, whose setters, such as `writer_class=`, are deprecated,
514
+ # see `IOStreams::Extension::Deprecations`.
515
+ #
516
+ # Registering `:encode` or `:none`, in any case, is deprecated, see RESERVED_EXTENSIONS, as is a name that IOStreams
517
+ # 3.0 does not register as it is, see .extension_name_deprecation: one that ends with a line break, or that is not in
518
+ # lower case. It still registers the extension, and reports a deprecation warning, see `IOStreams::Deprecation`.
519
+ #
387
520
  # Example:
388
521
  # # MyXls::Reader and MyXls::Writer must implement .open
389
522
  # register_extension(:xls, MyXls::Reader, MyXls::Writer)
390
523
  def self.register_extension(extension, reader_class, writer_class)
391
524
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
392
525
 
393
- @extensions[extension&.to_sym] = Extension.new(reader_class, writer_class)
526
+ if (reason = extension_name_deprecation(extension, RESERVED_EXTENSIONS))
527
+ Deprecation.warn("IOStreams.register_extension(#{extension.inspect}, ...) is deprecated, since #{reason}")
528
+ end
529
+ @extensions[extension&.to_sym] = Extension.registered(reader_class, writer_class)
530
+ end
531
+
532
+ # The names that IOStreams 3.0 reserves, so that registering an extension of one of them, in any case, which it raises
533
+ # ArgumentError for, is deprecated, see .register_extension, with why, and what to do instead.
534
+ RESERVED_EXTENSIONS = {
535
+ encode: "the encode stream is built in, which IOStreams 3.0 does not let an extension replace: register the " \
536
+ "reader and writer under another name instead, and apply them with #stream, or set the encoding of the " \
537
+ "text with #encoding",
538
+ none: "IOStreams 3.0 reserves :none, which stream(:none) took to apply no streams, register the stream under " \
539
+ "another name instead"
540
+ }.freeze
541
+ private_constant :RESERVED_EXTENSIONS
542
+
543
+ # Registers an extension that IOStreams builds in, as .register_extension does, whose methods of a Struct, other than
544
+ # `reader_class` and `writer_class`, are deprecated too, see `IOStreams::Extension::Deprecations`.
545
+ def self.register_built_in_extension(extension, reader_class, writer_class)
546
+ @extensions[extension] = Extension.registered(reader_class, writer_class, built_in: true)
394
547
  end
548
+ private_class_method :register_built_in_extension
395
549
 
396
550
  # De-Register a file extension
397
551
  #
398
- # Returns [Symbol] the extension removed, or nil if the extension was not registered
552
+ # Returns [IOStreams::Extension] the extension removed, or nil if the extension was not registered
553
+ #
554
+ # Deregistering `:encode`, in any case, is deprecated, since the encode stream is built in, which IOStreams 3.0 does
555
+ # not deregister, see BUILT_IN_STREAMS. It still removes it, so that `#encoding` then raises ArgumentError. So is a
556
+ # name that IOStreams 3.0 does not deregister as it is, see .extension_name_deprecation: one that ends with a line
557
+ # break, or that is not in lower case. Each reports a deprecation warning, see `IOStreams::Deprecation`.
399
558
  #
400
559
  # Example:
401
560
  # deregister_extension(:xls)
402
561
  def self.deregister_extension(extension)
403
562
  raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
404
563
 
564
+ if (reason = extension_name_deprecation(extension, BUILT_IN_STREAMS))
565
+ Deprecation.warn("IOStreams.deregister_extension(#{extension.inspect}) is deprecated, since #{reason}")
566
+ end
405
567
  @extensions.delete(extension.to_sym)
406
568
  end
407
569
 
408
- # Registered file extensions
570
+ # The streams that IOStreams 3.0 builds in, rather than registers, so that deregistering one of them, in any case,
571
+ # which it does not deregister, is deprecated, see .deregister_extension, with why, and what to do instead.
572
+ BUILT_IN_STREAMS = {
573
+ encode: "the encode stream is built in, which IOStreams 3.0 does not deregister, so remove it: IOStreams 3.0 " \
574
+ "applies the encode stream only where #encoding sets it"
575
+ }.freeze
576
+ private_constant :BUILT_IN_STREAMS
577
+
578
+ # Returns [String|nil] why the name of an extension, as the application supplied it to .register_extension or
579
+ # .deregister_extension, is deprecated, or nil when IOStreams 3.0 takes it as it is:
580
+ # - A name that ends with a line break, such as `"xz\n"`, which the check of the name accepts, while IOStreams 3.0
581
+ # raises ArgumentError for it.
582
+ # - A name of the supplied reasons, in any case, such as RESERVED_EXTENSIONS, with the reason that it gives.
583
+ # - A name that is not in lower case, such as `:XZ`, since IOStreams 3.0 registers and deregisters it in lower case,
584
+ # as `:xz`, which matches the file names that end in `.xz` in any case, as their extensions are compared.
585
+ def self.extension_name_deprecation(extension, reasons)
586
+ name = extension.to_s
587
+ lower = name.downcase
588
+ if name.end_with?("\n")
589
+ "IOStreams 3.0 raises ArgumentError for a name that ends with a line break, remove the line break"
590
+ elsif reasons.key?(lower.to_sym)
591
+ reasons[lower.to_sym]
592
+ elsif name != lower
593
+ "IOStreams 3.0 takes the name of an extension in lower case, as #{lower.to_sym.inspect}, which matches the file " \
594
+ "names that end in .#{lower}, in any case, use #{lower.to_sym.inspect} instead"
595
+ end
596
+ end
597
+ private_class_method :extension_name_deprecation
598
+
599
+ # Returns [Hash<Symbol, IOStreams::Extension>] the extension registered for each file extension, see
600
+ # .register_extension.
601
+ #
602
+ # Each extension answers `reader_class` and `writer_class`, as the format that IOStreams 3.0 returns for each of
603
+ # its built-in extensions does, such as `IOStreams::Gzip` for `:gz`. Its other methods of a Struct, such as `to_a`,
604
+ # `to_h` or `[]`, are deprecated for a built-in extension, and its setters, such as `writer_class=`, for every
605
+ # extension, see `IOStreams::Extension::Deprecations`.
409
606
  def self.extensions
410
607
  @extensions.dup
411
608
  end
412
609
 
610
+ # Returns [Array<Symbol>] the names of the formats of the rows and records of a file that `IOStreams::Stream#format`
611
+ # accepts, such as `:csv`, `:json` or `:fixed`, frozen. The list grows as formats are added, so check whether a name
612
+ # is included, rather than rely on its order or its length.
613
+ def self.tabular_formats
614
+ Tabular.registered_formats.freeze
615
+ end
616
+
413
617
  # Register a URI scheme and the path class that handles it
414
618
  #
415
619
  # Example:
@@ -428,7 +632,117 @@ module IOStreams
428
632
  @schemes[scheme_name&.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
429
633
  end
430
634
 
431
- Extension = Struct.new(:reader_class, :writer_class)
635
+ # The reader and writer classes registered for a file extension, see `IOStreams.register_extension`.
636
+ Extension = Struct.new(:reader_class, :writer_class) do
637
+ # Returns [Array<Symbol>] the options whose values must not be displayed, such as a passphrase, as declared
638
+ # by the reader and the writer with `sensitive_option_names`.
639
+ def sensitive_option_names
640
+ [reader_class, writer_class].flat_map do |klass|
641
+ (klass.sensitive_option_names if klass.respond_to?(:sensitive_option_names)) || []
642
+ end.uniq
643
+ end
644
+
645
+ # Returns [Hash] the options with the value of each sensitive option replaced with "[FILTERED]", so that
646
+ # they can be displayed, for example by `#inspect`.
647
+ #
648
+ # Sensitive options are those that the reader or the writer declares, see #sensitive_option_names, and,
649
+ # as a precaution, any option whose name suggests a secret, such as one that contains `password` or `token`, see
650
+ # `IOStreams::Utils.redact_options`, which owns that list.
651
+ def redact_options(options)
652
+ Utils.redact_options(options, sensitive_option_names)
653
+ end
654
+ end
655
+
656
+ class Extension
657
+ # Returns [IOStreams::Extension] a new extension of the supplied reader and writer classes, as
658
+ # `IOStreams.register_extension` registers it, whose setters are deprecated, or as IOStreams registers one that it
659
+ # builds in (built_in: true), whose other methods of a Struct are deprecated too, see Deprecations.
660
+ def self.registered(reader_class, writer_class, built_in: false)
661
+ new(reader_class, writer_class).tap { |extension| extension.send(:registered!, built_in) }
662
+ end
663
+
664
+ # Reports a deprecation warning, see `IOStreams::Deprecation`, for the methods of a Struct of a registered extension
665
+ # that IOStreams 3.0 does not answer, see .registered, and then does as before. It is prepended to the class, rather
666
+ # than extended onto each extension, so that an extension keeps its class, which JRuby's `Struct#eql?` compares.
667
+ #
668
+ # - The setters of a registered extension, such as `writer_class=`, and `[]=`, change the extension registered for
669
+ # the whole process, since `IOStreams.extensions` returns a copy of the registry that shares its extensions, which
670
+ # IOStreams 3.0 freezes. They still report one once it is no longer registered, such as one that
671
+ # `IOStreams.deregister_extension` returned, see #setter_deprecation, since IOStreams 3.0 raises for them there too.
672
+ # - For an extension that IOStreams builds in, such as `:gz`, IOStreams 3.0 returns the format that it registers,
673
+ # such as `IOStreams::Gzip`, rather than a Struct, see `IOStreams.extensions`. The format answers `reader_class`
674
+ # and `writer_class`, but none of the other methods of a Struct, such as #to_a, nor those of `Enumerable`, which
675
+ # call #each.
676
+ #
677
+ # An extension that is not registered, such as one that an application creates, reports nothing.
678
+ module Deprecations
679
+ # The methods that change an extension.
680
+ SETTERS = %i[reader_class= writer_class= []=].freeze
681
+
682
+ # The methods of a Struct that read an extension, other than `reader_class` and `writer_class`, and other than
683
+ # those that every object answers, such as `==` and `inspect`.
684
+ STRUCT_METHODS = %i[[] to_a values deconstruct to_h each each_pair members size length values_at select filter dig
685
+ deconstruct_keys].freeze
686
+
687
+ SETTERS.each do |name|
688
+ define_method(name) do |*args|
689
+ Deprecation.warn("##{name} #{setter_deprecation}") if @registered
690
+ super(*args)
691
+ end
692
+ end
693
+
694
+ STRUCT_METHODS.each do |name|
695
+ define_method(name) do |*args, &block|
696
+ if @built_in
697
+ Deprecation.warn("##{name} on a built-in extension of IOStreams.extensions is deprecated, since IOStreams " \
698
+ "3.0 returns its format, such as IOStreams::Gzip for :gz, rather than a Struct, use " \
699
+ "#reader_class and #writer_class instead")
700
+ end
701
+ super(*args, &block)
702
+ end
703
+ end
704
+
705
+ # Prints the extension for `pp` as it prints any other Struct, but reads each member with its reader, such as
706
+ # `reader_class`, rather than with #[], as `pp` does.
707
+ def pretty_print(printer)
708
+ printer.group(1, "#<struct #{self.class.name}", ">") do
709
+ printer.seplist(self.class.members, -> { printer.text(",") }) do |member|
710
+ printer.breakable
711
+ printer.text("#{member}=")
712
+ printer.group(1) do
713
+ printer.breakable("")
714
+ printer.pp(public_send(member))
715
+ end
716
+ end
717
+ end
718
+ end
719
+
720
+ private
721
+
722
+ # Marks this extension as registered, see .registered, by IOStreams itself when built_in.
723
+ def registered!(built_in)
724
+ @registered = true
725
+ @built_in = built_in
726
+ end
727
+
728
+ # Returns [String] why a setter of this extension, which was registered, see .registered, is deprecated, after the
729
+ # name of the setter: while it is registered, that the setter changes the extension registered for the whole
730
+ # process, and once it is not, such as after `IOStreams.deregister_extension` removed it, or another extension was
731
+ # registered in its place, that IOStreams 3.0 does not allow it either, since it freezes the extension that it
732
+ # registers, and returns the format of one that it builds in, such as `IOStreams::Gzip`, which has no setters.
733
+ def setter_deprecation
734
+ if IOStreams.extensions.each_value.any? { |extension| extension.equal?(self) }
735
+ "on an extension of IOStreams.extensions is deprecated, since it changes the registered extension for the " \
736
+ "whole process, which IOStreams 3.0 does not allow, use IOStreams.register_extension to change it instead"
737
+ else
738
+ "on an extension that is no longer registered, such as one that IOStreams.deregister_extension returned, is " \
739
+ "deprecated, since IOStreams 3.0 does not allow it on an extension that it registered, supply the classes " \
740
+ "to IOStreams.register_extension instead"
741
+ end
742
+ end
743
+ end
744
+ prepend Deprecations
745
+ end
432
746
 
433
747
  # Hold root paths
434
748
  @root_paths = {}
@@ -442,16 +756,16 @@ module IOStreams
442
756
  @schemes = {}
443
757
 
444
758
  # Register File extensions
445
- register_extension(:bz2, IOStreams::Bzip2::Reader, IOStreams::Bzip2::Writer)
446
- register_extension(:enc, IOStreams::SymmetricEncryption::Reader, IOStreams::SymmetricEncryption::Writer)
447
- register_extension(:gz, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
448
- register_extension(:gzip, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
449
- register_extension(:zip, IOStreams::Zip::Reader, IOStreams::Zip::Writer)
450
- register_extension(:pgp, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
451
- register_extension(:gpg, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
452
- register_extension(:xlsx, IOStreams::Xlsx::Reader, nil)
453
- register_extension(:xlsm, IOStreams::Xlsx::Reader, nil)
454
- register_extension(:encode, IOStreams::Encode::Reader, IOStreams::Encode::Writer)
759
+ register_built_in_extension(:bz2, IOStreams::Bzip2::Reader, IOStreams::Bzip2::Writer)
760
+ register_built_in_extension(:enc, IOStreams::SymmetricEncryption::Reader, IOStreams::SymmetricEncryption::Writer)
761
+ register_built_in_extension(:gz, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
762
+ register_built_in_extension(:gzip, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
763
+ register_built_in_extension(:zip, IOStreams::Zip::Reader, IOStreams::Zip::Writer)
764
+ register_built_in_extension(:pgp, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
765
+ register_built_in_extension(:gpg, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
766
+ register_built_in_extension(:xlsx, IOStreams::Xlsx::Reader, nil)
767
+ register_built_in_extension(:xlsm, IOStreams::Xlsx::Reader, nil)
768
+ register_built_in_extension(:encode, IOStreams::Encode::Reader, IOStreams::Encode::Writer)
455
769
 
456
770
  # Register Schemes
457
771
  #
@@ -467,4 +781,20 @@ module IOStreams
467
781
  register_scheme(:https, IOStreams::Paths::HTTP)
468
782
  register_scheme(:sftp, IOStreams::Paths::SFTP)
469
783
  register_scheme(:s3, IOStreams::Paths::S3)
784
+
785
+ # Reports a deprecation warning when the supplied version of Ruby is older than 3.3, which IOStreams 3.0 requires.
786
+ #
787
+ # Called once, as IOStreams is loaded, so that the warning names the line of the application that loaded it, such as
788
+ # its `Bundler.require`, see `IOStreams::Deprecation.warn_on_load`. It is only reported while deprecation warnings
789
+ # are on as IOStreams is loaded, such as with `RUBYOPT=-W:deprecated`, and RubyGems, which compares the versions, is
790
+ # loaded, which it is unless Ruby runs with `--disable-gems`.
791
+ def self.warn_unsupported_ruby(version = RUBY_VERSION)
792
+ return unless defined?(::Gem::Version) && ::Gem::Version.new(version) < ::Gem::Version.new("3.3")
793
+
794
+ Deprecation.warn_on_load("Ruby #{version} is deprecated, since IOStreams 3.0 requires Ruby 3.3 or later, so " \
795
+ "upgrade Ruby")
796
+ end
797
+ private_class_method :warn_unsupported_ruby
798
+
799
+ warn_unsupported_ruby
470
800
  end
@@ -19,7 +19,8 @@ module IOStreams
19
19
  # Create a delimited stream reader from the supplied input stream.
20
20
  #
21
21
  # Lines returned will be in the encoding of the input stream.
22
- # To change the encoding of returned lines, use IOStreams::Encode::Reader.
22
+ # To change the encoding of returned lines, use IOStreams::Encode::Reader. The lines of a file with a schema are
23
+ # read by `IOStreams::Line::Utf8Reader` instead, which takes them as UTF-8.
23
24
  #
24
25
  # Parameters
25
26
  # input_stream
@@ -0,0 +1,23 @@
1
+ module IOStreams
2
+ module Line
3
+ # Reads the lines of a file whose text is UTF-8, but is read without an encoding, as UTF-8 strings, rather than the
4
+ # binary strings that a file is read as without one: the lines of a file with a schema, see
5
+ # `IOStreams::Stream#schema` and `IOStreams::Builder#utf8_text?`, as its rows and records are read.
6
+ #
7
+ # Raises `IOStreams::Errors::InvalidEncoding` for a line that is not valid UTF-8, naming the line that the first
8
+ # invalid character is on, see `IOStreams::Utils.invalid_utf8`.
9
+ class Utf8Reader < Reader
10
+ def readline
11
+ first_line_number = line_number + 1
12
+ line = super
13
+ return line unless line&.encoding == ::Encoding::BINARY
14
+
15
+ # Each line is a String of its own, sliced from the data read, so it is taken as UTF-8 in place.
16
+ line.force_encoding(::Encoding::UTF_8)
17
+ return line if line.valid_encoding?
18
+
19
+ raise(IOStreams::Utils.invalid_utf8(line, line_number: first_line_number))
20
+ end
21
+ end
22
+ end
23
+ end