iostreams 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/config.md +13 -9
- data/docs/copy_files.md +20 -2
- data/docs/extensions.md +18 -4
- data/docs/formats.md +644 -16
- data/docs/path.md +18 -2
- data/docs/streams.md +54 -9
- data/docs/tutorial.md +14 -2
- data/docs/upgrading.md +470 -23
- data/lib/io_streams/builder/deprecations.rb +250 -0
- data/lib/io_streams/builder.rb +256 -39
- data/lib/io_streams/bzip2/reader.rb +18 -2
- data/lib/io_streams/bzip2/writer.rb +11 -2
- data/lib/io_streams/deprecation/methods.rb +51 -0
- data/lib/io_streams/deprecation/once.rb +27 -0
- data/lib/io_streams/deprecation/option_names.rb +26 -0
- data/lib/io_streams/deprecation/words.rb +42 -0
- data/lib/io_streams/deprecation.rb +259 -0
- data/lib/io_streams/encode/reader.rb +54 -3
- data/lib/io_streams/encode/writer.rb +19 -2
- data/lib/io_streams/encode.rb +99 -0
- data/lib/io_streams/errors.rb +51 -0
- data/lib/io_streams/gzip/reader.rb +11 -0
- data/lib/io_streams/io_streams.rb +365 -35
- data/lib/io_streams/line/reader.rb +2 -1
- data/lib/io_streams/line/utf8_reader.rb +23 -0
- data/lib/io_streams/path.rb +125 -35
- data/lib/io_streams/paths/file.rb +72 -9
- data/lib/io_streams/paths/http.rb +48 -2
- data/lib/io_streams/paths/matcher.rb +6 -3
- data/lib/io_streams/paths/s3.rb +180 -9
- data/lib/io_streams/paths/sftp.rb +47 -3
- data/lib/io_streams/paths.rb +23 -0
- data/lib/io_streams/pgp/reader.rb +18 -0
- data/lib/io_streams/pgp/writer.rb +16 -0
- data/lib/io_streams/pgp.rb +16 -2
- data/lib/io_streams/reader.rb +44 -0
- data/lib/io_streams/record/reader.rb +59 -68
- data/lib/io_streams/record/writer.rb +17 -10
- data/lib/io_streams/row/header_row.rb +21 -0
- data/lib/io_streams/row/reader.rb +52 -15
- data/lib/io_streams/row/writer.rb +64 -13
- data/lib/io_streams/schema/columns.rb +456 -0
- data/lib/io_streams/schema/constraints.rb +99 -0
- data/lib/io_streams/schema/field.rb +967 -0
- data/lib/io_streams/schema/json_number.rb +56 -0
- data/lib/io_streams/schema/keys.rb +78 -0
- data/lib/io_streams/schema/pattern.rb +257 -0
- data/lib/io_streams/schema/source.rb +31 -0
- data/lib/io_streams/schema.rb +616 -0
- data/lib/io_streams/stream.rb +836 -67
- data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
- data/lib/io_streams/tabular/header.rb +29 -0
- data/lib/io_streams/tabular/header_mapping.rb +332 -0
- data/lib/io_streams/tabular/mapping.rb +165 -0
- data/lib/io_streams/tabular/parser/array.rb +5 -0
- data/lib/io_streams/tabular/parser/base.rb +24 -0
- data/lib/io_streams/tabular/parser/fixed.rb +197 -0
- data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
- data/lib/io_streams/tabular/parser/hash.rb +10 -0
- data/lib/io_streams/tabular/parser/json.rb +17 -0
- data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
- data/lib/io_streams/tabular/schema_mapping.rb +361 -0
- data/lib/io_streams/tabular.rb +344 -30
- data/lib/io_streams/utils.rb +441 -6
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +27 -0
- data/lib/io_streams/xlsx/cells.rb +127 -0
- data/lib/io_streams/xlsx/reader.rb +56 -0
- data/lib/io_streams/xlsx/schema_reader.rb +90 -0
- data/lib/io_streams/xlsx/v3_writer.rb +16 -0
- data/lib/io_streams/zip/reader.rb +7 -0
- data/lib/io_streams/zip/writer.rb +24 -2
- data/lib/iostreams.rb +21 -17
- metadata +28 -2
|
@@ -35,41 +35,134 @@ module IOStreams
|
|
|
35
35
|
# # => "files/file.xls"
|
|
36
36
|
#
|
|
37
37
|
# For Files
|
|
38
|
-
# IOStreams.path('blah.zip').
|
|
39
|
-
# IOStreams.path('blah.zip').
|
|
40
|
-
# IOStreams.path('blah.zip').
|
|
41
|
-
# IOStreams.path('blah.zip').
|
|
38
|
+
# IOStreams.path('blah.zip').encoding('BINARY').each(:line) { |line| puts line }
|
|
39
|
+
# IOStreams.path('blah.zip').encoding('UTF-8').each(:line) { |line| puts line }
|
|
40
|
+
# IOStreams.path('blah.zip').encoding('UTF-8').each(:hash) { |hash| p hash }
|
|
41
|
+
# IOStreams.path('blah.zip').encoding('UTF-8').read
|
|
42
42
|
# IOStreams.path('blah.csv.zip').each(:line) { |line| puts line }
|
|
43
43
|
# IOStreams.path('blah.zip').option(:pgp, passphrase: 'receiver_passphrase').read
|
|
44
44
|
# IOStreams.path('blah.zip').stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
|
45
|
-
# IOStreams.path('blah.zip').stream(:zip).
|
|
45
|
+
# IOStreams.path('blah.zip').stream(:zip).encoding('BINARY').read
|
|
46
46
|
#
|
|
47
|
+
# A name that IOStreams v3.0 reads as another file or url, such as a local name starting with `~/`, a `file://` url, or
|
|
48
|
+
# an SFTP or HTTP url without a path, is deprecated, and reports a deprecation warning, see
|
|
49
|
+
# `IOStreams::Path.name_deprecation`.
|
|
47
50
|
def self.path(*elements, **args)
|
|
51
|
+
build_path(elements, args, :name_deprecation)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Returns [Hash] the options for creating the path of the url with `IOStreams.path(url, **options)`, with the value
|
|
55
|
+
# of each sensitive option, such as a password, replaced with "[FILTERED]", so that they can be displayed, for
|
|
56
|
+
# example by an application that stores a url and its options to create the path later.
|
|
57
|
+
#
|
|
58
|
+
# The path class of the url's scheme decides which options are sensitive, see `IOStreams::Path.redact_options`.
|
|
59
|
+
# Every value is replaced when the url is not valid, since then that class is not known, or when the class
|
|
60
|
+
# registered for the scheme, see `IOStreams.register_scheme`, cannot say which are sensitive.
|
|
61
|
+
#
|
|
62
|
+
# Example:
|
|
63
|
+
# IOStreams.redact_path_options("sftp://example.org/a.csv", username: "jack", password: "secret")
|
|
64
|
+
# # => {username: "jack", password: "[FILTERED]"}
|
|
65
|
+
def self.redact_path_options(url, options)
|
|
66
|
+
klass = Paths.path_class(url.to_s)
|
|
67
|
+
return klass.redact_options(options) if klass.respond_to?(:redact_options)
|
|
68
|
+
|
|
69
|
+
options.to_h { |name, _value| [name, Utils::FILTERED] }
|
|
70
|
+
rescue ArgumentError, ::URI::Error
|
|
71
|
+
options.to_h { |name, _value| [name, Utils::FILTERED] }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Returns [Hash<Symbol:Hash>] the options of each stream, as supplied to `IOStreams::Path#stream`, with the value of
|
|
75
|
+
# each sensitive option, such as a PGP passphrase, replaced with "[FILTERED]", so that they can be displayed.
|
|
76
|
+
#
|
|
77
|
+
# The extension registered for each stream decides which of its options are sensitive, see
|
|
78
|
+
# `IOStreams::Extension#redact_options`, as it does for `#inspect`. Every value of a stream that is not registered is
|
|
79
|
+
# replaced, as are options that are not a Hash.
|
|
80
|
+
#
|
|
81
|
+
# Example:
|
|
82
|
+
# IOStreams.redact_stream_options(pgp: {passphrase: "secret", recipient: "a@b.org"})
|
|
83
|
+
# # => {pgp: {passphrase: "[FILTERED]", recipient: "a@b.org"}}
|
|
84
|
+
def self.redact_stream_options(streams)
|
|
85
|
+
Utils.redact_streams(streams) { |name| @extensions[name.to_s.to_sym] }
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
# Returns [IOStreams::Path] the path of the supplied elements, joined into one name or url, created with the supplied
|
|
89
|
+
# arguments by the path class of its scheme, see `IOStreams::Paths.build`, reporting the deprecation warning, if any,
|
|
90
|
+
# that the supplied class method of that class gives for the name, see `IOStreams::Path.name_deprecation`, as the
|
|
91
|
+
# application supplied it.
|
|
92
|
+
def self.build_path(elements, args, deprecation)
|
|
48
93
|
return elements.first if (elements.size == 1) && args.empty? && elements.first.is_a?(IOStreams::Path)
|
|
49
94
|
|
|
50
|
-
|
|
51
|
-
path
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
95
|
+
name = Utils::PathName.join(*elements.collect(&:to_s))
|
|
96
|
+
path = Paths.build(name, **args)
|
|
97
|
+
warn_deprecated_name(path.class, deprecation, name)
|
|
98
|
+
path
|
|
99
|
+
end
|
|
100
|
+
private_class_method :build_path
|
|
101
|
+
|
|
102
|
+
# Reports the deprecation warning that the supplied class method of the path class gives for a name or url that the
|
|
103
|
+
# application supplied, if any: `:name_deprecation` when IOStreams v3.0 reads it as another file or url, or
|
|
104
|
+
# `:location_deprecation` when it reads it as another location, which the allowed paths compare, see
|
|
105
|
+
# `IOStreams::Path.name_deprecation`.
|
|
106
|
+
def self.warn_deprecated_name(klass, deprecation, name)
|
|
107
|
+
message = klass.public_send(deprecation, name) if klass.respond_to?(deprecation)
|
|
108
|
+
Deprecation.warn(message) if message
|
|
109
|
+
end
|
|
110
|
+
private_class_method :warn_deprecated_name
|
|
111
|
+
|
|
112
|
+
# Reports a deprecation warning for the name of the directory of a pattern, see .warn_deprecated_name. A pattern that
|
|
113
|
+
# is not a valid url, such as one with braces, is matched within the directory before its first pattern character,
|
|
114
|
+
# which the warnings do not apply to, see `IOStreams::Paths::Matcher`.
|
|
115
|
+
def self.warn_deprecated_pattern(pattern)
|
|
116
|
+
klass = Paths.path_class(pattern)
|
|
117
|
+
rescue ArgumentError, ::URI::Error
|
|
118
|
+
nil
|
|
119
|
+
else
|
|
120
|
+
warn_deprecated_name(klass, :name_deprecation, pattern)
|
|
121
|
+
end
|
|
122
|
+
private_class_method :warn_deprecated_pattern
|
|
123
|
+
|
|
124
|
+
# Returns [true|false] whether the object is a `Pathname`, which IOStreams v3.0 reads as the name of a file.
|
|
125
|
+
def self.pathname?(object)
|
|
126
|
+
return false unless defined?(::Pathname)
|
|
127
|
+
|
|
128
|
+
object.is_a?(::Pathname)
|
|
55
129
|
end
|
|
130
|
+
private_class_method :pathname?
|
|
56
131
|
|
|
57
132
|
# For an existing IO Stream
|
|
58
133
|
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').read
|
|
59
134
|
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').each(:line){ ... }
|
|
60
135
|
# IOStreams.stream(io).file_name('blah.csv.zip').each(:line) { ... }
|
|
61
136
|
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
|
137
|
+
#
|
|
138
|
+
# A `Pathname` is deprecated, since IOStreams v3.0 raises ArgumentError for it, as for a String: it reports a
|
|
139
|
+
# deprecation warning, see `IOStreams::Deprecation`.
|
|
62
140
|
def self.stream(io_stream)
|
|
63
141
|
return io_stream if io_stream.is_a?(Stream)
|
|
64
142
|
|
|
143
|
+
if pathname?(io_stream)
|
|
144
|
+
Deprecation.warn("a Pathname supplied to IOStreams.stream is deprecated, since v3.0 raises ArgumentError for it, " \
|
|
145
|
+
"as for a String: supply IOStreams.path(pathname) instead, and call #raw on it to read or " \
|
|
146
|
+
"write the data as it is stored")
|
|
147
|
+
end
|
|
65
148
|
Stream.new(io_stream)
|
|
66
149
|
end
|
|
67
150
|
|
|
68
151
|
# For processing by either a file name or an open IO stream.
|
|
152
|
+
#
|
|
153
|
+
# A `Pathname` is deprecated, which it reads and writes as an IO, since IOStreams v3.0 returns the path of the file
|
|
154
|
+
# that it names, which applies the streams of its file name: it reports a deprecation warning, see
|
|
155
|
+
# `IOStreams::Deprecation`. So is one supplied to `#copy_from`, `#copy_to` or `#move_to`, which call it.
|
|
69
156
|
def self.new(file_name_or_io)
|
|
70
157
|
return file_name_or_io if file_name_or_io.is_a?(Stream)
|
|
71
|
-
|
|
72
|
-
|
|
158
|
+
return path(file_name_or_io) if file_name_or_io.is_a?(String)
|
|
159
|
+
return stream(file_name_or_io) unless pathname?(file_name_or_io)
|
|
160
|
+
|
|
161
|
+
Deprecation.warn("a Pathname supplied to IOStreams.new, #copy_from, #copy_to or #move_to is deprecated, since " \
|
|
162
|
+
"v3.0 takes it as the name of a file, applying the streams that its file name implies, rather " \
|
|
163
|
+
"than as an IO: supply pathname.to_s instead, and call #raw on its path to read or write the " \
|
|
164
|
+
"data as it is stored")
|
|
165
|
+
Stream.new(file_name_or_io)
|
|
73
166
|
end
|
|
74
167
|
|
|
75
168
|
# Join the supplied path elements to a root path.
|
|
@@ -121,9 +214,14 @@ module IOStreams
|
|
|
121
214
|
# IOStreams.temp_file("export", ".csv") { |path| path.write("Hello World") }
|
|
122
215
|
#
|
|
123
216
|
# Note: The temp file is accessible even when it is not within the allowed paths, see `IOStreams.add_allowed_path`.
|
|
217
|
+
#
|
|
218
|
+
# The path reads and writes its data as it is stored, whatever its extension, while IOStreams v3.0 applies the streams
|
|
219
|
+
# that its extension implies, such as `.gz`, so reading or writing it without #raw, which reads and writes the data
|
|
220
|
+
# as it is stored on both, is deprecated when its extension implies streams, and reports a deprecation warning, see
|
|
221
|
+
# `IOStreams::Deprecation`.
|
|
124
222
|
def self.temp_file(basename, extension = "")
|
|
125
223
|
Utils.temp_file_name(basename, extension) do |file_name|
|
|
126
|
-
yield(Paths::File.new(file_name).send(:permit!).
|
|
224
|
+
yield(Paths::File.new(file_name).send(:permit!).send(:no_streams, temp_file: true))
|
|
127
225
|
end
|
|
128
226
|
end
|
|
129
227
|
|
|
@@ -219,6 +317,7 @@ module IOStreams
|
|
|
219
317
|
# "**.rb" "lib/song.rb" true
|
|
220
318
|
# "*" "dave/.profile" true
|
|
221
319
|
def self.each_child(pattern, case_sensitive: false, directories: false, hidden: false, &block)
|
|
320
|
+
warn_deprecated_pattern(pattern)
|
|
222
321
|
matcher = Paths::Matcher.new(nil, pattern, case_sensitive: case_sensitive, hidden: hidden)
|
|
223
322
|
|
|
224
323
|
# When the pattern includes an exact file name without any pattern characters
|
|
@@ -235,10 +334,20 @@ module IOStreams
|
|
|
235
334
|
end
|
|
236
335
|
|
|
237
336
|
# Add a named root path
|
|
337
|
+
#
|
|
338
|
+
# Returns [IOStreams::Path] the root path, which the whole process shares, see `IOStreams.join`. Changing it in place,
|
|
339
|
+
# such as with `#stream` or `#option`, whether through this path, a path supplied as the root, `IOStreams.root`,
|
|
340
|
+
# `IOStreams.roots`, or `IOStreams.join` without elements, is deprecated, and reports a deprecation warning, since v3.0
|
|
341
|
+
# keeps its own frozen copy of the root, which this method returns, and the others return copies of it, so that
|
|
342
|
+
# changing them no longer changes the root: change a copy of it instead, such as `IOStreams.root(:uploads).dup`, or a
|
|
343
|
+
# path joined to it, such as `IOStreams.join("a.csv", root: :uploads)`.
|
|
238
344
|
def self.add_root(root, *elements, **args)
|
|
239
345
|
raise(ArgumentError, "Invalid characters in root name #{root.inspect}") unless root.to_s =~ /\A\w+\Z/
|
|
240
346
|
|
|
241
|
-
|
|
347
|
+
path = path(*elements, **args)
|
|
348
|
+
# The path class of a scheme registered with `IOStreams.register_scheme` need not be an `IOStreams::Path`.
|
|
349
|
+
path.send(:root!, root.to_sym) if path.is_a?(Path)
|
|
350
|
+
@root_paths[root.to_sym] = path
|
|
242
351
|
end
|
|
243
352
|
|
|
244
353
|
def self.roots
|
|
@@ -280,7 +389,7 @@ module IOStreams
|
|
|
280
389
|
# * A local file could be replaced with a symbolic link after it is checked but before it is opened.
|
|
281
390
|
# Allowed paths do not prevent this, so do not allow paths where untrusted users can create files.
|
|
282
391
|
def self.add_allowed_path(*elements, **args)
|
|
283
|
-
location = allowed_location(
|
|
392
|
+
location = allowed_location(build_path(elements, args, :location_deprecation))
|
|
284
393
|
@allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths + [location]).uniq.freeze }
|
|
285
394
|
location
|
|
286
395
|
end
|
|
@@ -289,7 +398,7 @@ module IOStreams
|
|
|
289
398
|
#
|
|
290
399
|
# Returns [String] the normalized path that was removed.
|
|
291
400
|
def self.delete_allowed_path(*elements, **args)
|
|
292
|
-
location = allowed_location(
|
|
401
|
+
location = allowed_location(build_path(elements, args, :location_deprecation))
|
|
293
402
|
@allowed_paths_mutex.synchronize { @allowed_paths = (@allowed_paths - [location]).freeze }
|
|
294
403
|
location
|
|
295
404
|
end
|
|
@@ -305,7 +414,7 @@ module IOStreams
|
|
|
305
414
|
#
|
|
306
415
|
# Parameters: Same as `IOStreams.path`
|
|
307
416
|
def self.allowed_path?(*elements, **args)
|
|
308
|
-
|
|
417
|
+
build_path(elements, args, :location_deprecation).send(:allowed?)
|
|
309
418
|
end
|
|
310
419
|
|
|
311
420
|
def self.allowed_location(path)
|
|
@@ -348,22 +457,39 @@ module IOStreams
|
|
|
348
457
|
# Since the format is usually inferred from the file name, set this to true when the allowed columns
|
|
349
458
|
# restrict what an uploaded file can set, so that renaming the file to `.json` cannot bypass them.
|
|
350
459
|
#
|
|
351
|
-
# Default: false.
|
|
460
|
+
# Default: false.
|
|
352
461
|
#
|
|
353
462
|
# Example:
|
|
354
463
|
# IOStreams.enforce_column_restrictions = true
|
|
464
|
+
#
|
|
465
|
+
# Deprecated: it only applies to the column options, which are deprecated, see `IOStreams::Stream#each`, and which
|
|
466
|
+
# v3.0 removes, with this setting. Describe the columns of the file with `IOStreams::Stream#schema` instead, whose
|
|
467
|
+
# `fieldsMatch` and `constraints` apply to every input. It still applies to the column options, and reports a
|
|
468
|
+
# deprecation warning, see `IOStreams::Deprecation`, so keep it until they are replaced with a schema. The setting
|
|
469
|
+
# is held by `IOStreams::Tabular::HeaderMapping`, which applies it, and reads it without a deprecation warning.
|
|
355
470
|
def self.enforce_column_restrictions=(enforce)
|
|
356
471
|
raise(ArgumentError, "enforce_column_restrictions must be true or false") unless [true, false].include?(enforce)
|
|
357
472
|
|
|
358
|
-
|
|
473
|
+
Deprecation.warn("IOStreams.enforce_column_restrictions= #{ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION}")
|
|
474
|
+
Tabular::HeaderMapping.enforce_column_restrictions = enforce
|
|
359
475
|
end
|
|
360
476
|
|
|
361
477
|
# Returns [true|false] whether column restrictions apply to every input, see `IOStreams.enforce_column_restrictions=`.
|
|
478
|
+
#
|
|
479
|
+
# Deprecated, as `IOStreams.enforce_column_restrictions=` is. It reports a deprecation warning, see
|
|
480
|
+
# `IOStreams::Deprecation`.
|
|
362
481
|
def self.enforce_column_restrictions?
|
|
363
|
-
|
|
482
|
+
Deprecation.warn("IOStreams.enforce_column_restrictions? #{ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION}")
|
|
483
|
+
Tabular::HeaderMapping.enforce_column_restrictions?
|
|
364
484
|
end
|
|
365
485
|
|
|
366
|
-
|
|
486
|
+
# Why `IOStreams.enforce_column_restrictions=` and `IOStreams.enforce_column_restrictions?` are deprecated, and what
|
|
487
|
+
# to use instead.
|
|
488
|
+
ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION =
|
|
489
|
+
"is deprecated, since it only applies to the column options, such as allowed_columns:, which are deprecated: " \
|
|
490
|
+
"describe the columns of the file with #schema instead, whose fieldsMatch and constraints apply to every input, " \
|
|
491
|
+
"and then remove it".freeze
|
|
492
|
+
private_constant :ENFORCE_COLUMN_RESTRICTIONS_DEPRECATION
|
|
367
493
|
|
|
368
494
|
# Returns [Logger] the logger used by IOStreams for debug logging.
|
|
369
495
|
#
|
|
@@ -384,32 +510,110 @@ module IOStreams
|
|
|
384
510
|
|
|
385
511
|
# Register a file extension and the reader and writer streaming classes
|
|
386
512
|
#
|
|
513
|
+
# Returns [IOStreams::Extension] the extension registered, whose setters, such as `writer_class=`, are deprecated,
|
|
514
|
+
# see `IOStreams::Extension::Deprecations`.
|
|
515
|
+
#
|
|
516
|
+
# Registering `:encode` or `:none`, in any case, is deprecated, see RESERVED_EXTENSIONS, as is a name that IOStreams
|
|
517
|
+
# 3.0 does not register as it is, see .extension_name_deprecation: one that ends with a line break, or that is not in
|
|
518
|
+
# lower case. It still registers the extension, and reports a deprecation warning, see `IOStreams::Deprecation`.
|
|
519
|
+
#
|
|
387
520
|
# Example:
|
|
388
521
|
# # MyXls::Reader and MyXls::Writer must implement .open
|
|
389
522
|
# register_extension(:xls, MyXls::Reader, MyXls::Writer)
|
|
390
523
|
def self.register_extension(extension, reader_class, writer_class)
|
|
391
524
|
raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.nil? || extension.to_s =~ /\A\w+\Z/
|
|
392
525
|
|
|
393
|
-
|
|
526
|
+
if (reason = extension_name_deprecation(extension, RESERVED_EXTENSIONS))
|
|
527
|
+
Deprecation.warn("IOStreams.register_extension(#{extension.inspect}, ...) is deprecated, since #{reason}")
|
|
528
|
+
end
|
|
529
|
+
@extensions[extension&.to_sym] = Extension.registered(reader_class, writer_class)
|
|
530
|
+
end
|
|
531
|
+
|
|
532
|
+
# The names that IOStreams 3.0 reserves, so that registering an extension of one of them, in any case, which it raises
|
|
533
|
+
# ArgumentError for, is deprecated, see .register_extension, with why, and what to do instead.
|
|
534
|
+
RESERVED_EXTENSIONS = {
|
|
535
|
+
encode: "the encode stream is built in, which IOStreams 3.0 does not let an extension replace: register the " \
|
|
536
|
+
"reader and writer under another name instead, and apply them with #stream, or set the encoding of the " \
|
|
537
|
+
"text with #encoding",
|
|
538
|
+
none: "IOStreams 3.0 reserves :none, which stream(:none) took to apply no streams, register the stream under " \
|
|
539
|
+
"another name instead"
|
|
540
|
+
}.freeze
|
|
541
|
+
private_constant :RESERVED_EXTENSIONS
|
|
542
|
+
|
|
543
|
+
# Registers an extension that IOStreams builds in, as .register_extension does, whose methods of a Struct, other than
|
|
544
|
+
# `reader_class` and `writer_class`, are deprecated too, see `IOStreams::Extension::Deprecations`.
|
|
545
|
+
def self.register_built_in_extension(extension, reader_class, writer_class)
|
|
546
|
+
@extensions[extension] = Extension.registered(reader_class, writer_class, built_in: true)
|
|
394
547
|
end
|
|
548
|
+
private_class_method :register_built_in_extension
|
|
395
549
|
|
|
396
550
|
# De-Register a file extension
|
|
397
551
|
#
|
|
398
|
-
# Returns [
|
|
552
|
+
# Returns [IOStreams::Extension] the extension removed, or nil if the extension was not registered
|
|
553
|
+
#
|
|
554
|
+
# Deregistering `:encode`, in any case, is deprecated, since the encode stream is built in, which IOStreams 3.0 does
|
|
555
|
+
# not deregister, see BUILT_IN_STREAMS. It still removes it, so that `#encoding` then raises ArgumentError. So is a
|
|
556
|
+
# name that IOStreams 3.0 does not deregister as it is, see .extension_name_deprecation: one that ends with a line
|
|
557
|
+
# break, or that is not in lower case. Each reports a deprecation warning, see `IOStreams::Deprecation`.
|
|
399
558
|
#
|
|
400
559
|
# Example:
|
|
401
560
|
# deregister_extension(:xls)
|
|
402
561
|
def self.deregister_extension(extension)
|
|
403
562
|
raise(ArgumentError, "Invalid extension #{extension.inspect}") unless extension.to_s =~ /\A\w+\Z/
|
|
404
563
|
|
|
564
|
+
if (reason = extension_name_deprecation(extension, BUILT_IN_STREAMS))
|
|
565
|
+
Deprecation.warn("IOStreams.deregister_extension(#{extension.inspect}) is deprecated, since #{reason}")
|
|
566
|
+
end
|
|
405
567
|
@extensions.delete(extension.to_sym)
|
|
406
568
|
end
|
|
407
569
|
|
|
408
|
-
#
|
|
570
|
+
# The streams that IOStreams 3.0 builds in, rather than registers, so that deregistering one of them, in any case,
|
|
571
|
+
# which it does not deregister, is deprecated, see .deregister_extension, with why, and what to do instead.
|
|
572
|
+
BUILT_IN_STREAMS = {
|
|
573
|
+
encode: "the encode stream is built in, which IOStreams 3.0 does not deregister, so remove it: IOStreams 3.0 " \
|
|
574
|
+
"applies the encode stream only where #encoding sets it"
|
|
575
|
+
}.freeze
|
|
576
|
+
private_constant :BUILT_IN_STREAMS
|
|
577
|
+
|
|
578
|
+
# Returns [String|nil] why the name of an extension, as the application supplied it to .register_extension or
|
|
579
|
+
# .deregister_extension, is deprecated, or nil when IOStreams 3.0 takes it as it is:
|
|
580
|
+
# - A name that ends with a line break, such as `"xz\n"`, which the check of the name accepts, while IOStreams 3.0
|
|
581
|
+
# raises ArgumentError for it.
|
|
582
|
+
# - A name of the supplied reasons, in any case, such as RESERVED_EXTENSIONS, with the reason that it gives.
|
|
583
|
+
# - A name that is not in lower case, such as `:XZ`, since IOStreams 3.0 registers and deregisters it in lower case,
|
|
584
|
+
# as `:xz`, which matches the file names that end in `.xz` in any case, as their extensions are compared.
|
|
585
|
+
def self.extension_name_deprecation(extension, reasons)
|
|
586
|
+
name = extension.to_s
|
|
587
|
+
lower = name.downcase
|
|
588
|
+
if name.end_with?("\n")
|
|
589
|
+
"IOStreams 3.0 raises ArgumentError for a name that ends with a line break, remove the line break"
|
|
590
|
+
elsif reasons.key?(lower.to_sym)
|
|
591
|
+
reasons[lower.to_sym]
|
|
592
|
+
elsif name != lower
|
|
593
|
+
"IOStreams 3.0 takes the name of an extension in lower case, as #{lower.to_sym.inspect}, which matches the file " \
|
|
594
|
+
"names that end in .#{lower}, in any case, use #{lower.to_sym.inspect} instead"
|
|
595
|
+
end
|
|
596
|
+
end
|
|
597
|
+
private_class_method :extension_name_deprecation
|
|
598
|
+
|
|
599
|
+
# Returns [Hash<Symbol, IOStreams::Extension>] the extension registered for each file extension, see
|
|
600
|
+
# .register_extension.
|
|
601
|
+
#
|
|
602
|
+
# Each extension answers `reader_class` and `writer_class`, as the format that IOStreams 3.0 returns for each of
|
|
603
|
+
# its built-in extensions does, such as `IOStreams::Gzip` for `:gz`. Its other methods of a Struct, such as `to_a`,
|
|
604
|
+
# `to_h` or `[]`, are deprecated for a built-in extension, and its setters, such as `writer_class=`, for every
|
|
605
|
+
# extension, see `IOStreams::Extension::Deprecations`.
|
|
409
606
|
def self.extensions
|
|
410
607
|
@extensions.dup
|
|
411
608
|
end
|
|
412
609
|
|
|
610
|
+
# Returns [Array<Symbol>] the names of the formats of the rows and records of a file that `IOStreams::Stream#format`
|
|
611
|
+
# accepts, such as `:csv`, `:json` or `:fixed`, frozen. The list grows as formats are added, so check whether a name
|
|
612
|
+
# is included, rather than rely on its order or its length.
|
|
613
|
+
def self.tabular_formats
|
|
614
|
+
Tabular.registered_formats.freeze
|
|
615
|
+
end
|
|
616
|
+
|
|
413
617
|
# Register a URI scheme and the path class that handles it
|
|
414
618
|
#
|
|
415
619
|
# Example:
|
|
@@ -428,7 +632,117 @@ module IOStreams
|
|
|
428
632
|
@schemes[scheme_name&.to_sym] || raise(ArgumentError, "Unknown Scheme type: #{scheme_name.inspect}")
|
|
429
633
|
end
|
|
430
634
|
|
|
431
|
-
|
|
635
|
+
# The reader and writer classes registered for a file extension, see `IOStreams.register_extension`.
|
|
636
|
+
Extension = Struct.new(:reader_class, :writer_class) do
|
|
637
|
+
# Returns [Array<Symbol>] the options whose values must not be displayed, such as a passphrase, as declared
|
|
638
|
+
# by the reader and the writer with `sensitive_option_names`.
|
|
639
|
+
def sensitive_option_names
|
|
640
|
+
[reader_class, writer_class].flat_map do |klass|
|
|
641
|
+
(klass.sensitive_option_names if klass.respond_to?(:sensitive_option_names)) || []
|
|
642
|
+
end.uniq
|
|
643
|
+
end
|
|
644
|
+
|
|
645
|
+
# Returns [Hash] the options with the value of each sensitive option replaced with "[FILTERED]", so that
|
|
646
|
+
# they can be displayed, for example by `#inspect`.
|
|
647
|
+
#
|
|
648
|
+
# Sensitive options are those that the reader or the writer declares, see #sensitive_option_names, and,
|
|
649
|
+
# as a precaution, any option whose name suggests a secret, such as one that contains `password` or `token`, see
|
|
650
|
+
# `IOStreams::Utils.redact_options`, which owns that list.
|
|
651
|
+
def redact_options(options)
|
|
652
|
+
Utils.redact_options(options, sensitive_option_names)
|
|
653
|
+
end
|
|
654
|
+
end
|
|
655
|
+
|
|
656
|
+
class Extension
|
|
657
|
+
# Returns [IOStreams::Extension] a new extension of the supplied reader and writer classes, as
|
|
658
|
+
# `IOStreams.register_extension` registers it, whose setters are deprecated, or as IOStreams registers one that it
|
|
659
|
+
# builds in (built_in: true), whose other methods of a Struct are deprecated too, see Deprecations.
|
|
660
|
+
def self.registered(reader_class, writer_class, built_in: false)
|
|
661
|
+
new(reader_class, writer_class).tap { |extension| extension.send(:registered!, built_in) }
|
|
662
|
+
end
|
|
663
|
+
|
|
664
|
+
# Reports a deprecation warning, see `IOStreams::Deprecation`, for the methods of a Struct of a registered extension
|
|
665
|
+
# that IOStreams 3.0 does not answer, see .registered, and then does as before. It is prepended to the class, rather
|
|
666
|
+
# than extended onto each extension, so that an extension keeps its class, which JRuby's `Struct#eql?` compares.
|
|
667
|
+
#
|
|
668
|
+
# - The setters of a registered extension, such as `writer_class=`, and `[]=`, change the extension registered for
|
|
669
|
+
# the whole process, since `IOStreams.extensions` returns a copy of the registry that shares its extensions, which
|
|
670
|
+
# IOStreams 3.0 freezes. They still report one once it is no longer registered, such as one that
|
|
671
|
+
# `IOStreams.deregister_extension` returned, see #setter_deprecation, since IOStreams 3.0 raises for them there too.
|
|
672
|
+
# - For an extension that IOStreams builds in, such as `:gz`, IOStreams 3.0 returns the format that it registers,
|
|
673
|
+
# such as `IOStreams::Gzip`, rather than a Struct, see `IOStreams.extensions`. The format answers `reader_class`
|
|
674
|
+
# and `writer_class`, but none of the other methods of a Struct, such as #to_a, nor those of `Enumerable`, which
|
|
675
|
+
# call #each.
|
|
676
|
+
#
|
|
677
|
+
# An extension that is not registered, such as one that an application creates, reports nothing.
|
|
678
|
+
module Deprecations
|
|
679
|
+
# The methods that change an extension.
|
|
680
|
+
SETTERS = %i[reader_class= writer_class= []=].freeze
|
|
681
|
+
|
|
682
|
+
# The methods of a Struct that read an extension, other than `reader_class` and `writer_class`, and other than
|
|
683
|
+
# those that every object answers, such as `==` and `inspect`.
|
|
684
|
+
STRUCT_METHODS = %i[[] to_a values deconstruct to_h each each_pair members size length values_at select filter dig
|
|
685
|
+
deconstruct_keys].freeze
|
|
686
|
+
|
|
687
|
+
SETTERS.each do |name|
|
|
688
|
+
define_method(name) do |*args|
|
|
689
|
+
Deprecation.warn("##{name} #{setter_deprecation}") if @registered
|
|
690
|
+
super(*args)
|
|
691
|
+
end
|
|
692
|
+
end
|
|
693
|
+
|
|
694
|
+
STRUCT_METHODS.each do |name|
|
|
695
|
+
define_method(name) do |*args, &block|
|
|
696
|
+
if @built_in
|
|
697
|
+
Deprecation.warn("##{name} on a built-in extension of IOStreams.extensions is deprecated, since IOStreams " \
|
|
698
|
+
"3.0 returns its format, such as IOStreams::Gzip for :gz, rather than a Struct, use " \
|
|
699
|
+
"#reader_class and #writer_class instead")
|
|
700
|
+
end
|
|
701
|
+
super(*args, &block)
|
|
702
|
+
end
|
|
703
|
+
end
|
|
704
|
+
|
|
705
|
+
# Prints the extension for `pp` as it prints any other Struct, but reads each member with its reader, such as
|
|
706
|
+
# `reader_class`, rather than with #[], as `pp` does.
|
|
707
|
+
def pretty_print(printer)
|
|
708
|
+
printer.group(1, "#<struct #{self.class.name}", ">") do
|
|
709
|
+
printer.seplist(self.class.members, -> { printer.text(",") }) do |member|
|
|
710
|
+
printer.breakable
|
|
711
|
+
printer.text("#{member}=")
|
|
712
|
+
printer.group(1) do
|
|
713
|
+
printer.breakable("")
|
|
714
|
+
printer.pp(public_send(member))
|
|
715
|
+
end
|
|
716
|
+
end
|
|
717
|
+
end
|
|
718
|
+
end
|
|
719
|
+
|
|
720
|
+
private
|
|
721
|
+
|
|
722
|
+
# Marks this extension as registered, see .registered, by IOStreams itself when built_in.
|
|
723
|
+
def registered!(built_in)
|
|
724
|
+
@registered = true
|
|
725
|
+
@built_in = built_in
|
|
726
|
+
end
|
|
727
|
+
|
|
728
|
+
# Returns [String] why a setter of this extension, which was registered, see .registered, is deprecated, after the
|
|
729
|
+
# name of the setter: while it is registered, that the setter changes the extension registered for the whole
|
|
730
|
+
# process, and once it is not, such as after `IOStreams.deregister_extension` removed it, or another extension was
|
|
731
|
+
# registered in its place, that IOStreams 3.0 does not allow it either, since it freezes the extension that it
|
|
732
|
+
# registers, and returns the format of one that it builds in, such as `IOStreams::Gzip`, which has no setters.
|
|
733
|
+
def setter_deprecation
|
|
734
|
+
if IOStreams.extensions.each_value.any? { |extension| extension.equal?(self) }
|
|
735
|
+
"on an extension of IOStreams.extensions is deprecated, since it changes the registered extension for the " \
|
|
736
|
+
"whole process, which IOStreams 3.0 does not allow, use IOStreams.register_extension to change it instead"
|
|
737
|
+
else
|
|
738
|
+
"on an extension that is no longer registered, such as one that IOStreams.deregister_extension returned, is " \
|
|
739
|
+
"deprecated, since IOStreams 3.0 does not allow it on an extension that it registered, supply the classes " \
|
|
740
|
+
"to IOStreams.register_extension instead"
|
|
741
|
+
end
|
|
742
|
+
end
|
|
743
|
+
end
|
|
744
|
+
prepend Deprecations
|
|
745
|
+
end
|
|
432
746
|
|
|
433
747
|
# Hold root paths
|
|
434
748
|
@root_paths = {}
|
|
@@ -442,16 +756,16 @@ module IOStreams
|
|
|
442
756
|
@schemes = {}
|
|
443
757
|
|
|
444
758
|
# Register File extensions
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
759
|
+
register_built_in_extension(:bz2, IOStreams::Bzip2::Reader, IOStreams::Bzip2::Writer)
|
|
760
|
+
register_built_in_extension(:enc, IOStreams::SymmetricEncryption::Reader, IOStreams::SymmetricEncryption::Writer)
|
|
761
|
+
register_built_in_extension(:gz, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
|
|
762
|
+
register_built_in_extension(:gzip, IOStreams::Gzip::Reader, IOStreams::Gzip::Writer)
|
|
763
|
+
register_built_in_extension(:zip, IOStreams::Zip::Reader, IOStreams::Zip::Writer)
|
|
764
|
+
register_built_in_extension(:pgp, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
|
|
765
|
+
register_built_in_extension(:gpg, IOStreams::Pgp::Reader, IOStreams::Pgp::Writer)
|
|
766
|
+
register_built_in_extension(:xlsx, IOStreams::Xlsx::Reader, nil)
|
|
767
|
+
register_built_in_extension(:xlsm, IOStreams::Xlsx::Reader, nil)
|
|
768
|
+
register_built_in_extension(:encode, IOStreams::Encode::Reader, IOStreams::Encode::Writer)
|
|
455
769
|
|
|
456
770
|
# Register Schemes
|
|
457
771
|
#
|
|
@@ -467,4 +781,20 @@ module IOStreams
|
|
|
467
781
|
register_scheme(:https, IOStreams::Paths::HTTP)
|
|
468
782
|
register_scheme(:sftp, IOStreams::Paths::SFTP)
|
|
469
783
|
register_scheme(:s3, IOStreams::Paths::S3)
|
|
784
|
+
|
|
785
|
+
# Reports a deprecation warning when the supplied version of Ruby is older than 3.3, which IOStreams 3.0 requires.
|
|
786
|
+
#
|
|
787
|
+
# Called once, as IOStreams is loaded, so that the warning names the line of the application that loaded it, such as
|
|
788
|
+
# its `Bundler.require`, see `IOStreams::Deprecation.warn_on_load`. It is only reported while deprecation warnings
|
|
789
|
+
# are on as IOStreams is loaded, such as with `RUBYOPT=-W:deprecated`, and RubyGems, which compares the versions, is
|
|
790
|
+
# loaded, which it is unless Ruby runs with `--disable-gems`.
|
|
791
|
+
def self.warn_unsupported_ruby(version = RUBY_VERSION)
|
|
792
|
+
return unless defined?(::Gem::Version) && ::Gem::Version.new(version) < ::Gem::Version.new("3.3")
|
|
793
|
+
|
|
794
|
+
Deprecation.warn_on_load("Ruby #{version} is deprecated, since IOStreams 3.0 requires Ruby 3.3 or later, so " \
|
|
795
|
+
"upgrade Ruby")
|
|
796
|
+
end
|
|
797
|
+
private_class_method :warn_unsupported_ruby
|
|
798
|
+
|
|
799
|
+
warn_unsupported_ruby
|
|
470
800
|
end
|
|
@@ -19,7 +19,8 @@ module IOStreams
|
|
|
19
19
|
# Create a delimited stream reader from the supplied input stream.
|
|
20
20
|
#
|
|
21
21
|
# Lines returned will be in the encoding of the input stream.
|
|
22
|
-
# To change the encoding of returned lines, use IOStreams::Encode::Reader.
|
|
22
|
+
# To change the encoding of returned lines, use IOStreams::Encode::Reader. The lines of a file with a schema are
|
|
23
|
+
# read by `IOStreams::Line::Utf8Reader` instead, which takes them as UTF-8.
|
|
23
24
|
#
|
|
24
25
|
# Parameters
|
|
25
26
|
# input_stream
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module IOStreams
|
|
2
|
+
module Line
|
|
3
|
+
# Reads the lines of a file whose text is UTF-8, but is read without an encoding, as UTF-8 strings, rather than the
|
|
4
|
+
# binary strings that a file is read as without one: the lines of a file with a schema, see
|
|
5
|
+
# `IOStreams::Stream#schema` and `IOStreams::Builder#utf8_text?`, as its rows and records are read.
|
|
6
|
+
#
|
|
7
|
+
# Raises `IOStreams::Errors::InvalidEncoding` for a line that is not valid UTF-8, naming the line that the first
|
|
8
|
+
# invalid character is on, see `IOStreams::Utils.invalid_utf8`.
|
|
9
|
+
class Utf8Reader < Reader
|
|
10
|
+
def readline
|
|
11
|
+
first_line_number = line_number + 1
|
|
12
|
+
line = super
|
|
13
|
+
return line unless line&.encoding == ::Encoding::BINARY
|
|
14
|
+
|
|
15
|
+
# Each line is a String of its own, sliced from the data read, so it is taken as UTF-8 in place.
|
|
16
|
+
line.force_encoding(::Encoding::UTF_8)
|
|
17
|
+
return line if line.valid_encoding?
|
|
18
|
+
|
|
19
|
+
raise(IOStreams::Utils.invalid_utf8(line, line_number: first_line_number))
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|