iostreams 2.1.0 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. checksums.yaml +4 -4
  2. data/docs/config.md +13 -9
  3. data/docs/copy_files.md +20 -2
  4. data/docs/extensions.md +18 -4
  5. data/docs/formats.md +644 -16
  6. data/docs/path.md +18 -2
  7. data/docs/streams.md +54 -9
  8. data/docs/tutorial.md +14 -2
  9. data/docs/upgrading.md +470 -23
  10. data/lib/io_streams/builder/deprecations.rb +250 -0
  11. data/lib/io_streams/builder.rb +256 -39
  12. data/lib/io_streams/bzip2/reader.rb +18 -2
  13. data/lib/io_streams/bzip2/writer.rb +11 -2
  14. data/lib/io_streams/deprecation/methods.rb +51 -0
  15. data/lib/io_streams/deprecation/once.rb +27 -0
  16. data/lib/io_streams/deprecation/option_names.rb +26 -0
  17. data/lib/io_streams/deprecation/words.rb +42 -0
  18. data/lib/io_streams/deprecation.rb +259 -0
  19. data/lib/io_streams/encode/reader.rb +54 -3
  20. data/lib/io_streams/encode/writer.rb +19 -2
  21. data/lib/io_streams/encode.rb +99 -0
  22. data/lib/io_streams/errors.rb +51 -0
  23. data/lib/io_streams/gzip/reader.rb +11 -0
  24. data/lib/io_streams/io_streams.rb +365 -35
  25. data/lib/io_streams/line/reader.rb +2 -1
  26. data/lib/io_streams/line/utf8_reader.rb +23 -0
  27. data/lib/io_streams/path.rb +125 -35
  28. data/lib/io_streams/paths/file.rb +72 -9
  29. data/lib/io_streams/paths/http.rb +48 -2
  30. data/lib/io_streams/paths/matcher.rb +6 -3
  31. data/lib/io_streams/paths/s3.rb +180 -9
  32. data/lib/io_streams/paths/sftp.rb +47 -3
  33. data/lib/io_streams/paths.rb +23 -0
  34. data/lib/io_streams/pgp/reader.rb +18 -0
  35. data/lib/io_streams/pgp/writer.rb +16 -0
  36. data/lib/io_streams/pgp.rb +16 -2
  37. data/lib/io_streams/reader.rb +44 -0
  38. data/lib/io_streams/record/reader.rb +59 -68
  39. data/lib/io_streams/record/writer.rb +17 -10
  40. data/lib/io_streams/row/header_row.rb +21 -0
  41. data/lib/io_streams/row/reader.rb +52 -15
  42. data/lib/io_streams/row/writer.rb +64 -13
  43. data/lib/io_streams/schema/columns.rb +456 -0
  44. data/lib/io_streams/schema/constraints.rb +99 -0
  45. data/lib/io_streams/schema/field.rb +967 -0
  46. data/lib/io_streams/schema/json_number.rb +56 -0
  47. data/lib/io_streams/schema/keys.rb +78 -0
  48. data/lib/io_streams/schema/pattern.rb +257 -0
  49. data/lib/io_streams/schema/source.rb +31 -0
  50. data/lib/io_streams/schema.rb +616 -0
  51. data/lib/io_streams/stream.rb +836 -67
  52. data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
  53. data/lib/io_streams/tabular/header.rb +29 -0
  54. data/lib/io_streams/tabular/header_mapping.rb +332 -0
  55. data/lib/io_streams/tabular/mapping.rb +165 -0
  56. data/lib/io_streams/tabular/parser/array.rb +5 -0
  57. data/lib/io_streams/tabular/parser/base.rb +24 -0
  58. data/lib/io_streams/tabular/parser/fixed.rb +197 -0
  59. data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
  60. data/lib/io_streams/tabular/parser/hash.rb +10 -0
  61. data/lib/io_streams/tabular/parser/json.rb +17 -0
  62. data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
  63. data/lib/io_streams/tabular/schema_mapping.rb +361 -0
  64. data/lib/io_streams/tabular.rb +344 -30
  65. data/lib/io_streams/utils.rb +441 -6
  66. data/lib/io_streams/version.rb +1 -1
  67. data/lib/io_streams/writer.rb +27 -0
  68. data/lib/io_streams/xlsx/cells.rb +127 -0
  69. data/lib/io_streams/xlsx/reader.rb +56 -0
  70. data/lib/io_streams/xlsx/schema_reader.rb +90 -0
  71. data/lib/io_streams/xlsx/v3_writer.rb +16 -0
  72. data/lib/io_streams/zip/reader.rb +7 -0
  73. data/lib/io_streams/zip/writer.rb +24 -2
  74. data/lib/iostreams.rb +21 -17
  75. metadata +28 -2
@@ -5,6 +5,81 @@ module IOStreams
5
5
  module Utils
6
6
  MAX_TEMP_FILE_NAME_ATTEMPTS = 5
7
7
 
8
+ # Replaces the value of a sensitive option for display, see .redact_options.
9
+ FILTERED = "[FILTERED]".freeze
10
+
11
+ # Option names whose values are hidden as a precaution, even when they are not declared sensitive.
12
+ # Matched against the name in lower case without `_` or `-`, see .redact_options.
13
+ SENSITIVE_NAME = /passphrase|password|secret|token|credential|auth|apikey|privatekey/
14
+ private_constant :SENSITIVE_NAME
15
+
16
+ # Returns [Hash] the options with the value of each sensitive option replaced with FILTERED, so that they can be
17
+ # displayed, for example by `#inspect`, or in a web interface.
18
+ #
19
+ # An option is sensitive when its name is one of the supplied `sensitive_names`, or, as a precaution, contains
20
+ # `passphrase`, `password`, `secret`, `token`, `credential`, `auth`, `apikey` or `privatekey`. Names are compared in
21
+ # lower case, ignoring `_` and `-`, so that `:identity_key` matches the ssh option "IdentityKey", and the header
22
+ # "X-Api-Key" contains `apikey`. The options in a Hash value, such as `ssh_options:` or `client:`, are redacted the
23
+ # same way, as are those in an Array value, whether its elements are Hashes or `[name, value]` pairs.
24
+ def self.redact_options(options, sensitive_names = [])
25
+ redact_hash(options, sensitive_names.map { |name| normalize_option_name(name) })
26
+ end
27
+
28
+ # Returns [Hash] the options of each stream, by its name, such as those supplied to `IOStreams::Stream#stream`, with
29
+ # the value of each sensitive option replaced with FILTERED, as the extension that the block returns for the name of
30
+ # the stream decides, see `IOStreams::Extension#redact_options`.
31
+ #
32
+ # Every value of a stream is replaced when the block returns nil, since no extension can say which are sensitive,
33
+ # and options that are not a Hash are replaced whole, since they cannot say which of their values are sensitive.
34
+ # Options that are nil are kept.
35
+ def self.redact_streams(streams)
36
+ streams.to_h do |name, options|
37
+ next [name, options] if options.nil?
38
+ next [name, FILTERED] unless options.is_a?(Hash)
39
+
40
+ extension = yield(name)
41
+ [name, extension ? extension.redact_options(options) : options.transform_values { FILTERED }]
42
+ end
43
+ end
44
+
45
+ def self.redact_hash(options, sensitive)
46
+ options.to_h { |name, value| [name, redact_option(name, value, sensitive)] }
47
+ end
48
+ private_class_method :redact_hash
49
+
50
+ # Returns the value of the named option for display, see .redact_options.
51
+ def self.redact_option(name, value, sensitive)
52
+ key = normalize_option_name(name)
53
+ return FILTERED if sensitive.include?(key) || key.match?(SENSITIVE_NAME)
54
+
55
+ redact_value(value, sensitive)
56
+ end
57
+ private_class_method :redact_option
58
+
59
+ # Returns the value with the options within it redacted, when it is a Hash or an Array, see .redact_options.
60
+ def self.redact_value(value, sensitive)
61
+ case value
62
+ when Hash
63
+ redact_hash(value, sensitive)
64
+ when Array
65
+ value.map do |element|
66
+ if element.is_a?(Array) && element.size == 2 && (element.first.is_a?(String) || element.first.is_a?(Symbol))
67
+ [element.first, redact_option(element.first, element.last, sensitive)]
68
+ else
69
+ redact_value(element, sensitive)
70
+ end
71
+ end
72
+ else
73
+ value
74
+ end
75
+ end
76
+ private_class_method :redact_value
77
+
78
+ def self.normalize_option_name(name)
79
+ name.to_s.downcase.delete("_-")
80
+ end
81
+ private_class_method :normalize_option_name
82
+
8
83
  # Lazy load dependent gem so that it remains a soft dependency.
9
84
  def self.load_soft_dependency(gem_name, stream_type, require_name = gem_name)
10
85
  require require_name
@@ -12,17 +87,47 @@ module IOStreams
12
87
  raise(LoadError, "Please install the gem '#{gem_name}' to support #{stream_type}. #{e.message}")
13
88
  end
14
89
 
15
- # Log a warning for options that a stream ignores, until they raise `ArgumentError` in v3.0.
16
- def self.warn_unknown_options(unknown, stream, direction, valid)
90
+ # Log a warning for options that a stream ignores when reading (type: :reader) or writing (:writer): those that the
91
+ # class of the other direction takes, other_names, which v3.0 also accepts, and ignores, and the others, which v3.0
92
+ # raises `ArgumentError` for. A path supplies other_names, and the stream as it was set, see
93
+ # `IOStreams::Builder#taken_options`; without them, as when a class is opened directly, each option is one that
94
+ # v3.0 raises for, since its class takes only its own options.
95
+ #
96
+ # Those of neither direction are deprecated, and report a deprecation warning when they are set, see
97
+ # `IOStreams::Builder#option`.
98
+ def self.warn_unknown_options(unknown, stream, type, valid, other_names = [])
17
99
  return if unknown.empty?
18
100
 
19
- names = unknown.keys.map(&:inspect).join(", ")
101
+ other = unknown.keys & other_names
102
+ neither = unknown.keys - other
103
+ direction = type == :reader ? "reading" : "writing"
104
+ unless other.empty?
105
+ IOStreams.logger&.warn(
106
+ "Ignoring #{other.map(&:inspect).join(', ')} when #{direction} a #{stream.inspect} stream, since " \
107
+ "#{other.size == 1 ? 'it only applies' : 'they only apply'} when #{type == :reader ? 'writing' : 'reading'}."
108
+ )
109
+ end
110
+ return if neither.empty?
111
+
20
112
  IOStreams.logger&.warn(
21
- "Ignoring unknown #{unknown.size == 1 ? 'option' : 'options'} #{names} when #{direction} a #{stream.inspect} " \
22
- "stream. Valid options: #{valid.map(&:inspect).join(', ')}. In v3.0 this will raise ArgumentError."
113
+ "Ignoring unknown #{neither.size == 1 ? 'option' : 'options'} #{neither.map(&:inspect).join(', ')} when " \
114
+ "#{direction} a #{stream.inspect} stream. Valid options: #{valid.map(&:inspect).join(', ')}. " \
115
+ "In v3.0 this will raise ArgumentError."
23
116
  )
24
117
  end
25
118
 
119
+ # Returns [String] the string as UTF-8 text that can be displayed or logged, such as the name of a file, which is not
120
+ # always valid UTF-8, since a file system stores the bytes that a program gave it: a binary string is read as UTF-8,
121
+ # and each byte that is not valid UTF-8 is shown as `\xHH`.
122
+ #
123
+ # For example the name of a file in Latin-1, `caf\xE9.csv`, is displayed as "caf\\xE9.csv".
124
+ def self.display_text(string)
125
+ text = string.encoding == Encoding::BINARY ? string.dup.force_encoding(Encoding::UTF_8) : string
126
+ return text if text.valid_encoding?
127
+
128
+ text.scrub { |bytes| bytes.unpack("C*").map { |byte| format("\\x%02X", byte) }.join }
129
+ end
130
+
26
131
  # Helper method: Returns [true|false] if a value is blank?
27
132
  def self.blank?(value)
28
133
  return true if value.nil?
@@ -31,6 +136,106 @@ module IOStreams
31
136
  value.respond_to?(:empty?) ? value.empty? : !value
32
137
  end
33
138
 
139
+ # Text that is blank, see .blank_text?: empty, or only whitespace, as Rails' `String#blank?` matches it.
140
+ BLANK_TEXT = /\A[[:space:]]*\z/
141
+ private_constant :BLANK_TEXT
142
+
143
+ # The pattern of BLANK_TEXT in each encoding that it cannot be matched against, such as UTF-16, see .blank_text?.
144
+ @encoded_blank_texts = {}
145
+ @encoded_blank_texts_mutex = Mutex.new
146
+
147
+ # The rules below of what is blank apply to the rows and records read and written with a schema, see
148
+ # `IOStreams::Tabular::SchemaMapping`, as IOStreams 3.0 applies them to every file, while a file without a schema
149
+ # keeps those of .blank?.
150
+
151
+ # Returns [true|false] whether the text is blank, by the rules of Rails' `String#blank?`: empty, or only whitespace,
152
+ # which is any Unicode whitespace, such as a no-break space (U+00A0) or an ideographic space (U+3000), not only a
153
+ # space, tab, or line ending. Text in an encoding that is not ASCII compatible, such as UTF-16, is matched in its own
154
+ # encoding, as Rails matches it.
155
+ #
156
+ # Raises ArgumentError for text whose bytes are not valid in its encoding, as Rails does.
157
+ def self.blank_text?(text)
158
+ text.empty? || BLANK_TEXT.match?(text)
159
+ rescue Encoding::CompatibilityError
160
+ encoded_blank_text(text.encoding).match?(text)
161
+ end
162
+
163
+ # Returns [true|false] whether a value within a row or record is blank, so that it holds no data: nil, or text that
164
+ # is blank, see .blank_text?. Any other value is data, such as false, 0, an empty Array, or text whose bytes are not
165
+ # valid in its encoding, since it is not text that is known to be blank.
166
+ def self.blank_value?(value)
167
+ value.nil? || (value.is_a?(String) && value.valid_encoding? && blank_text?(value))
168
+ end
169
+
170
+ # Returns [true|false] whether a row or record is blank, so that it holds no data: nil, or an Array or Hash without
171
+ # values, or whose every value is blank, see .blank_value?, such as the row of a line of CSV of only commas, or a
172
+ # record of JSON whose every value is null.
173
+ def self.blank_row?(row)
174
+ case row
175
+ when nil then true
176
+ when ::Hash then row.each_value.all? { |value| blank_value?(value) }
177
+ when ::Array then row.all? { |value| blank_value?(value) }
178
+ else false
179
+ end
180
+ end
181
+
182
+ # Returns [Regexp] the pattern of blank text in the supplied encoding, see .blank_text?, created once for each one.
183
+ def self.encoded_blank_text(encoding)
184
+ @encoded_blank_texts_mutex.synchronize do
185
+ @encoded_blank_texts[encoding] ||=
186
+ Regexp.new(BLANK_TEXT.source.encode(encoding), BLANK_TEXT.options | Regexp::FIXEDENCODING)
187
+ end
188
+ end
189
+ private_class_method :encoded_blank_text
190
+
191
+ # Returns [String] the text, taken as UTF-8 when it is binary, as IOStreams 2 reads a file without an encoding,
192
+ # see `IOStreams::Stream#encoding`, such as a file read with a schema, whose text is UTF-8, see
193
+ # `IOStreams::Stream#schema`. Text in any other encoding is returned as it is.
194
+ #
195
+ # Raises `IOStreams::Errors::InvalidEncoding` when it is not valid UTF-8.
196
+ def self.utf8(text)
197
+ return text unless text.encoding == ::Encoding::BINARY
198
+
199
+ utf8 = text.dup.force_encoding(::Encoding::UTF_8)
200
+ return utf8 if utf8.valid_encoding?
201
+
202
+ raise(invalid_utf8(utf8))
203
+ end
204
+
205
+ # Returns [IOStreams::Errors::InvalidEncoding] the error for text taken as UTF-8 that is not valid UTF-8, naming
206
+ # its first invalid character, and the line that it is on when the number of the line that the text starts on is
207
+ # supplied, counting the line breaks within the text, such as within a quoted CSV value.
208
+ def self.invalid_utf8(utf8, line_number: nil)
209
+ invalid = nil
210
+ utf8.each_char do |char|
211
+ unless char.valid_encoding?
212
+ invalid = char.b
213
+ break
214
+ end
215
+ line_number += 1 if line_number && char == "\n"
216
+ end
217
+ on_line = line_number ? " on line #{line_number}" : ""
218
+ IOStreams::Errors::InvalidEncoding.new(
219
+ "#{invalid.inspect} is not valid UTF-8#{on_line}. Set the encoding of the file with #encoding",
220
+ line_number: line_number,
221
+ error_char: invalid,
222
+ source_encoding: ::Encoding::BINARY,
223
+ destination_encoding: ::Encoding::UTF_8
224
+ )
225
+ end
226
+
227
+ # Returns a frozen copy of the value, whose Hashes, Arrays and Strings are frozen copies too, so that holding a
228
+ # setting, such as a schema, does not freeze the objects that the caller supplied, and the caller changing them
229
+ # does not change the setting. Any other value, such as a Symbol, is returned as it is.
230
+ def self.frozen_copy(value)
231
+ case value
232
+ when Hash then value.to_h { |key, item| [frozen_copy(key), frozen_copy(item)] }.freeze
233
+ when Array then value.map { |item| frozen_copy(item) }.freeze
234
+ when String then value.frozen? ? value : value.dup.freeze
235
+ else value
236
+ end
237
+ end
238
+
34
239
  # Yields the path to a temporary file_name.
35
240
  #
36
241
  # File is deleted upon completion if present.
@@ -64,8 +269,237 @@ module IOStreams
64
269
  end
65
270
  end
66
271
 
272
+ module PathName
273
+ # The byte of the separator of the names in a path, `/`, which every encoding that a path can be in, those that are
274
+ # ASCII-compatible, holds as this byte, see .join.
275
+ SEPARATOR = "/".ord
276
+ private_constant :SEPARATOR
277
+
278
+ # The byte of the `.` that starts the extension of a name, see .extname.
279
+ DOT = ".".ord
280
+ private_constant :DOT
281
+
282
+ # Whether `::File.join`, `::File.dirname`, `::File.extname` and `::File.basename` keep the bytes of a name that is
283
+ # not valid UTF-8, as CRuby does, rather than changing them, as JRuby does, see
284
+ # https://github.com/jruby/jruby/issues/9880, such as the name of a file in Latin-1 that SFTP lists as bytes, or
285
+ # that is labelled UTF-8, so that the path no longer names the file. Found by what they do, so that
286
+ # IOStreams uses them again once JRuby keeps the bytes.
287
+ FILE_KEEPS_NAMES =
288
+ begin
289
+ name = "a\xE9".b
290
+ label = name.dup.force_encoding(Encoding::UTF_8)
291
+ ::File.join(name, name).b == "a\xE9/a\xE9".b && ::File.dirname("#{name}/x").b == name &&
292
+ ::File.extname("x.".b + name).b == ".a\xE9".b && ::File.basename("x/#{label}").b == name
293
+ end
294
+
295
+ # Returns [String] the supplied names joined into a path, as `::File.join` joins Strings: with one `/` between
296
+ # each name and the next, whose separators decide which are kept, keeping the bytes and the encoding of each name,
297
+ # where `::File.join` does, see FILE_KEEPS_NAMES, otherwise by .join_names.
298
+ def self.join(*names)
299
+ FILE_KEEPS_NAMES ? ::File.join(*names) : join_names(names)
300
+ end
301
+
302
+ # Returns [String] the directory of the supplied path, as `::File.dirname` returns it, see .join.
303
+ def self.dirname(path)
304
+ FILE_KEEPS_NAMES ? ::File.dirname(path) : dirname_of(path)
305
+ end
306
+
307
+ # Returns [String] the extension of the last name of the supplied path, as `::File.extname` returns it, see .join.
308
+ def self.extname(path)
309
+ FILE_KEEPS_NAMES ? ::File.extname(path) : extname_of(path)
310
+ end
311
+
312
+ # Returns [String] the last name of the supplied path, as `::File.basename` returns it, without the supplied
313
+ # suffix when it ends with it, see .join.
314
+ def self.basename(path, suffix = nil)
315
+ return basename_of(path, suffix) unless FILE_KEEPS_NAMES
316
+
317
+ suffix.nil? ? ::File.basename(path) : ::File.basename(path, suffix)
318
+ end
319
+
320
+ # Returns [String] the supplied Strings joined into a path, as CRuby's `File.join` joins them on POSIX, by their
321
+ # bytes: each name after the first is joined to the path so far with one `/`, unless the path ends with `/`, when
322
+ # it is added as it is, or the name starts with `/`, when the path loses its trailing `/` before it. The path starts
323
+ # in the encoding of the first name, and takes that of a name whose text needs it, as for `String#<<`, which raises
324
+ # `Encoding::CompatibilityError` for names that have no encoding in common.
325
+ #
326
+ # Used where `::File.join` changes names, see .join, and public, so that its tests compare it with CRuby's.
327
+ def self.join_names(names)
328
+ joined = String.new(encoding: names.first&.encoding || Encoding::BINARY)
329
+ names.each_with_index do |name, index|
330
+ if index.positive?
331
+ tail = trailing_separators(joined)
332
+ if name.getbyte(0) == SEPARATOR
333
+ joined = joined.byteslice(0, tail)
334
+ elsif tail == joined.bytesize
335
+ joined << "/"
336
+ end
337
+ end
338
+ joined << name
339
+ end
340
+ joined
341
+ end
342
+
343
+ # Returns [String] the directory of the supplied path, as CRuby's `File.dirname` returns it on POSIX, by its bytes:
344
+ # the path before the separators of its last name, `/` for a path of only separators, or `.` for a single name, in
345
+ # the encoding of the path.
346
+ #
347
+ # Used where `::File.dirname` changes names, see .dirname, and public, so that its tests compare it with CRuby's.
348
+ def self.dirname_of(path)
349
+ root = leading_separators(path)
350
+ start = root > 1 ? root - 1 : 0
351
+ finish = last_separator(path, root) || root
352
+ return String.new(".", encoding: path.encoding) if finish == start
353
+
354
+ path.byteslice(start, finish - start)
355
+ end
356
+
357
+ # Returns [String] the extension of the last name of the supplied path, as CRuby's `File.extname` returns it on
358
+ # POSIX, by its bytes: from its last `.`, unless only dots start the name, in the encoding of the path, or an
359
+ # empty String when it has none.
360
+ #
361
+ # Used where `::File.extname` changes names, see .extname, and public, so that its tests compare it with CRuby's.
362
+ def self.extname_of(path)
363
+ separator = last_separator(path, 0)
364
+ position = separator ? separator + 1 : 0
365
+ position += 1 while position < path.bytesize && path.getbyte(position) == SEPARATOR
366
+ position += 1 while position < path.bytesize && path.getbyte(position) == DOT
367
+ dot = nil
368
+ while position < path.bytesize
369
+ byte = path.getbyte(position)
370
+ break if byte == SEPARATOR
371
+
372
+ dot = position if byte == DOT
373
+ position += 1
374
+ end
375
+ return "".b if dot.nil?
376
+
377
+ path.byteslice(dot, position - dot)
378
+ end
379
+
380
+ # Returns [String] the last name of the supplied path, as CRuby's `File.basename` returns it on POSIX, by its
381
+ # bytes: the name after its last separators, without the separators at its end, `/` for a path of only
382
+ # separators, in the encoding of the path. The supplied suffix is removed when the name ends with it, `.*` removes
383
+ # the extension of the name, see .extname, and a character followed by `*` removes the name from the last of that
384
+ # character, unless that would leave nothing. A suffix without an encoding in common with the path is not
385
+ # removed.
386
+ #
387
+ # Raises ArgumentError, as CRuby does, when the first character of the suffix, or for a character followed by
388
+ # `*`, a character of the name, is not valid in the encoding that they have in common.
389
+ #
390
+ # Used where `::File.basename` changes names, see .basename, and public, so that its tests compare it with CRuby's.
391
+ def self.basename_of(path, suffix = nil)
392
+ return path.dup if path.empty?
393
+
394
+ start = leading_separators(path)
395
+ return path.byteslice(start - 1, 1) if start == path.bytesize
396
+
397
+ separator = last_separator(path, start)
398
+ if separator
399
+ start = separator
400
+ start += 1 while path.getbyte(start) == SEPARATOR
401
+ end
402
+ length = trailing_separators(path) - start
403
+ encoding = suffix && Encoding.compatible?(path, suffix)
404
+ kept = encoding ? without_suffix(path, start, length, suffix.dup.force_encoding(encoding)) : 0
405
+ path.byteslice(start, kept.positive? ? kept : length)
406
+ end
407
+
408
+ # Returns [Integer] the number of bytes of the name of the supplied length, from the supplied position of the
409
+ # supplied path, that are kept once the supplied suffix is removed, see .basename_of, or 0 when the suffix is not
410
+ # removed, as CRuby's `rmext` does.
411
+ def self.without_suffix(path, start, length, suffix)
412
+ return 0 if suffix.empty?
413
+
414
+ first = valid_character(suffix[0])
415
+ if suffix.bytesize == first.bytesize + 1 && suffix.getbyte(first.bytesize) == "*".ord
416
+ return stem_length(path, start, length) if first == "."
417
+
418
+ name = valid_character(path.byteslice(start, length).force_encoding(suffix.encoding))
419
+ index = name.rindex(first)
420
+ return index ? name[0, index].bytesize : length
421
+ end
422
+ return length if length < suffix.bytesize
423
+
424
+ from = start + length - suffix.bytesize
425
+ return 0 unless character_start?(path, from, suffix.encoding)
426
+
427
+ path.byteslice(from, suffix.bytesize).b == suffix.b ? length - suffix.bytesize : 0
428
+ end
429
+
430
+ # Returns [String] the supplied text.
431
+ #
432
+ # Raises ArgumentError when it is not valid in its encoding, as CRuby does when it reads its characters.
433
+ def self.valid_character(text)
434
+ return text if text.valid_encoding?
435
+
436
+ raise(ArgumentError, "invalid byte sequence in #{text.encoding}")
437
+ end
438
+ private_class_method :valid_character
439
+ private_class_method :without_suffix
440
+
441
+ # Returns [Integer] the number of bytes of the name of the supplied length, from the supplied position of the
442
+ # supplied path, before its extension, see .extname_of: before its last `.`, unless only dots start the name.
443
+ def self.stem_length(path, start, length)
444
+ position = start
445
+ position += 1 while position - start < length && path.getbyte(position) == DOT
446
+ dot = nil
447
+ while position - start < length
448
+ dot = position if path.getbyte(position) == DOT
449
+ position += 1
450
+ end
451
+ dot ? dot - start : length
452
+ end
453
+ private_class_method :stem_length
454
+
455
+ # Returns [true|false] whether a character of the supplied path, read in the supplied encoding, starts at the
456
+ # supplied position: always for a single byte encoding, and for UTF-8 unless its byte continues a character.
457
+ def self.character_start?(path, position, encoding)
458
+ return true unless encoding == Encoding::UTF_8
459
+
460
+ !(0x80..0xBF).cover?(path.getbyte(position))
461
+ end
462
+ private_class_method :character_start?
463
+
464
+ # Returns [Integer] the number of bytes of the supplied path before the separators at its end.
465
+ def self.trailing_separators(path)
466
+ tail = path.bytesize
467
+ tail -= 1 while tail.positive? && path.getbyte(tail - 1) == SEPARATOR
468
+ tail
469
+ end
470
+ private_class_method :trailing_separators
471
+
472
+ # Returns [Integer] the number of separators at the start of the supplied path.
473
+ def self.leading_separators(path)
474
+ count = 0
475
+ count += 1 while count < path.bytesize && path.getbyte(count) == SEPARATOR
476
+ count
477
+ end
478
+ private_class_method :leading_separators
479
+
480
+ # Returns [Integer] the position of the first of the last separators of the supplied path, from the supplied
481
+ # position on, that a name follows, or nil when it has none, as CRuby's `strrdirsep` finds it.
482
+ def self.last_separator(path, from)
483
+ last = nil
484
+ position = from
485
+ while position < path.bytesize
486
+ if path.getbyte(position) == SEPARATOR
487
+ separator = position
488
+ position += 1 while position < path.bytesize && path.getbyte(position) == SEPARATOR
489
+ break if position >= path.bytesize
490
+
491
+ last = separator
492
+ else
493
+ position += 1
494
+ end
495
+ end
496
+ last
497
+ end
498
+ private_class_method :last_separator
499
+ end
500
+
67
501
  class URI
68
- attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
502
+ attr_reader :scheme, :hostname, :path, :user, :password, :port, :query, :fragment
69
503
 
70
504
  def initialize(url)
71
505
  url = url.gsub(" ", "%20")
@@ -76,6 +510,7 @@ module IOStreams
76
510
  @user = uri.user
77
511
  @password = uri.password
78
512
  @port = uri.port
513
+ @fragment = uri.fragment
79
514
  return unless uri.query
80
515
 
81
516
  @query = {}
@@ -1,3 +1,3 @@
1
1
  module IOStreams
2
- VERSION = "2.1.0".freeze
2
+ VERSION = "2.2.0".freeze
3
3
  end
@@ -1,5 +1,8 @@
1
1
  module IOStreams
2
2
  class Writer
3
+ # Answers the names of the options that this writer takes in IOStreams 3.0, and those that it replaces.
4
+ extend Deprecation::OptionNames
5
+
3
6
  # Returns [Array<Symbol>] the names of the options this writer accepts,
4
7
  # or [nil] when the writer does not declare them.
5
8
  #
@@ -9,6 +12,30 @@ module IOStreams
9
12
  nil
10
13
  end
11
14
 
15
+ # Returns [Array<Symbol>] the names of the options whose values must not be displayed, such as a passphrase,
16
+ # so that `#inspect` on a path does not display them, see `IOStreams::Extension#redact_options`.
17
+ def self.sensitive_option_names
18
+ []
19
+ end
20
+
21
+ # The methods of an IO that write, which every stream that v3.0 supplies to a writer block answers, see
22
+ # .v3_stream_methods.
23
+ V3_WRITE_METHODS = %i[write << print puts printf].freeze
24
+
25
+ # Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a writer block, in place of the one
26
+ # that this writer yields, answers, when it does not answer all of the methods of that one, otherwise nil, for a
27
+ # writer whose stream answers the same methods in both versions, see `IOStreams::Reader.v3_stream_methods`, whose
28
+ # parameters and block it takes too.
29
+ def self.v3_stream_methods(_options)
30
+ nil
31
+ end
32
+
33
+ # Returns [Hash<Symbol:String>|nil] what the application uses in place of each method of the stream that this writer
34
+ # yields that v3.0 removes, see `IOStreams::Reader.v3_stream_advice`.
35
+ def self.v3_stream_advice
36
+ nil
37
+ end
38
+
12
39
  # When a Writer does not support streams, we copy the stream to a local temp file
13
40
  # and then pass that filename in for this reader.
14
41
  def self.stream(output_stream, **args, &block)
@@ -0,0 +1,127 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "date"
4
+
5
+ module IOStreams
6
+ module Xlsx
7
+ # The class of the number format of each cell of a worksheet, from Excel's own number formats, and the text of a cell
8
+ # of a date or a time, as IOStreams 3.0 types them, so that a spreadsheet read with a schema, see
9
+ # `IOStreams::Xlsx::SchemaReader`, holds the same dates and times in IOStreams 2.2 and 3.0, rather than as creek
10
+ # classifies them, which takes every custom format with a letter of a date or a time as a date and time, such as
11
+ # `yyyy-mm-dd`, or `0.00" days"`:
12
+ #
13
+ # Number format | Text of the cell
14
+ # a date | its date in ISO 8601, such as `2026-10-25`
15
+ # a date and time | the time of its clock in UTC, such as `2026-10-25T09:30:00Z`, since Excel has no zone
16
+ # a time of day | the time of its clock, such as `09:30:00`
17
+ module Cells
18
+ # The seconds of a day.
19
+ DAY = 86_400
20
+
21
+ # The first day of a time of day, as Active Record holds the value of a `time` column.
22
+ TIME_OF_DAY = [2000, 1, 1].freeze
23
+
24
+ # The number of the day after the last date that Excel holds, 9999-12-31, since 1899-12-30.
25
+ LAST_DAY = 2_958_466
26
+
27
+ # The built-in number formats of Excel of dates, times of day, and dates with times, by their ids.
28
+ BUILT_IN_FORMATS = {
29
+ 14 => :date, 15 => :date, 16 => :date, 17 => :date, 18 => :time, 19 => :time, 20 => :time, 21 => :time,
30
+ 22 => :date_time, 45 => :time, 46 => :time, 47 => :time
31
+ }.freeze
32
+
33
+ # The parts of a custom format code that are not its tokens: quoted text, an escaped character, and a section in
34
+ # brackets other than an elapsed time, such as a color or a locale. A section never holds another `[`, so a match
35
+ # stops there, rather than scan the rest of the code again from each `[` of a workbook's code that never closes.
36
+ LITERALS = /"[^"]*"|\\.|_.|\*.|\[(?![hms]+\])[^\[\]]*\]/i
37
+
38
+ # Returns [Array<Symbol|nil>] the class of the number format of each style of a workbook, by the index of the style,
39
+ # as a cell names it with its `s` attribute: `:date`, `:time` or `:date_time`, or nil for a number, from Excel's own
40
+ # number formats, see BUILT_IN_FORMATS and .custom_format, rather than as creek finds them.
41
+ #
42
+ # Parameters:
43
+ # styles: [Nokogiri::XML::Document|nil]
44
+ # The styles of the workbook, `xl/styles.xml`, or nil for a workbook without them.
45
+ def self.number_formats(styles)
46
+ return [] if styles.nil?
47
+
48
+ custom = styles.css("styleSheet numFmts numFmt").to_h do |format|
49
+ [format["numFmtId"].to_i, custom_format(format["formatCode"].to_s)]
50
+ end
51
+ styles.css("styleSheet cellXfs xf").map do |style|
52
+ id = style["numFmtId"]&.to_i
53
+ BUILT_IN_FORMATS.fetch(id) { custom[id] } if id
54
+ end
55
+ end
56
+
57
+ # Returns [Symbol|nil] the class of a custom format code, by the first of its sections: `:date` with only a year,
58
+ # month or day, `:time` with only hours, minutes or seconds, `:date_time` with both, otherwise nil, for a number.
59
+ # An `m` is minutes after hours or before seconds, otherwise the month.
60
+ def self.custom_format(code)
61
+ tokens = code.split(";").first.to_s.gsub(LITERALS, "")
62
+ time = tokens.match?(%r{[hs]|am/pm|a/p}i)
63
+ date = tokens.match?(/[yd]/i) || (!time && tokens.match?(/m/i))
64
+ if date && time then :date_time
65
+ elsif date then :date
66
+ elsif time then :time
67
+ end
68
+ end
69
+
70
+ # Returns [String|nil] the text of a cell of a number type, the text of its `<v>` element, by the class of its
71
+ # number format, see .number_formats, with the first day of the workbook, see `Creek::Sheet#converter_options`, as
72
+ # IOStreams 3.0 types it, so that a field of its type reads it:
73
+ # a date in ISO 8601, such as "2026-10-25"
74
+ # a date and time its time in UTC, such as "2026-10-25T09:30:00Z", with its milliseconds when it has any
75
+ # a time the time of its clock, such as "09:30:00", or of a number of a day or more, its date
76
+ # and time, as a date and time
77
+ #
78
+ # Returns nil for a number of any other format, or one of a date or time format that is not a date that Excel
79
+ # holds, such as a negative number, which Excel shows as `####`, which is a number.
80
+ def self.date_text(text, style, base_date)
81
+ days = days(text) if style
82
+ return unless days
83
+
84
+ case style
85
+ when :date then (base_date + days.floor).iso8601
86
+ when :date_time then Schema::Field.any_text(date_time(days, base_date))
87
+ when :time then days < 1 ? time_of_day_text(time_of_day(days)) : Schema::Field.any_text(date_time(days, base_date))
88
+ end
89
+ rescue ArgumentError, ZeroDivisionError
90
+ nil
91
+ end
92
+
93
+ # Returns [Rational|nil] the days since the first day of the workbook of the text of a number, or nil for one
94
+ # outside the dates that Excel holds, from its first day up to 9999-12-31, such as a negative number, or one of a
95
+ # huge exponent, which is checked as a Float before it becomes a Rational, so that its cost is bounded.
96
+ def self.days(text)
97
+ days = Float(text)
98
+ Rational(text) if days >= 0 && days < LAST_DAY
99
+ end
100
+
101
+ # Returns [String] the time of the clock of a time of day, such as "09:30:00", with the milliseconds that Excel
102
+ # holds when it has any, such as "09:30:00.500", as a `time` field of IOStreams 3.0 writes it.
103
+ def self.time_of_day_text(time)
104
+ time.nsec.zero? ? time.strftime("%H:%M:%S") : time.strftime("%H:%M:%S.%3N")
105
+ end
106
+
107
+ # Returns [Time] the time, in UTC, of the clock of the days since the first day of the workbook, to the millisecond
108
+ # that Excel holds.
109
+ def self.date_time(days, base_date)
110
+ date = base_date + days.floor
111
+ Time.utc(date.year, date.month, date.day) + seconds(days)
112
+ end
113
+
114
+ # Returns [Time] the time of day of the fraction of a day, on 2000-01-01 in UTC, also when the fraction rounds to
115
+ # a whole day, which is midnight.
116
+ def self.time_of_day(days)
117
+ Time.utc(*TIME_OF_DAY) + (seconds(days) % DAY)
118
+ end
119
+
120
+ # Returns [Rational] the seconds of the fraction of the day, to the millisecond.
121
+ def self.seconds(days)
122
+ ((days - days.floor) * DAY).round(3)
123
+ end
124
+ private_class_method :days, :time_of_day_text, :date_time, :time_of_day, :seconds
125
+ end
126
+ end
127
+ end