iostreams 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/config.md +13 -9
- data/docs/copy_files.md +20 -2
- data/docs/extensions.md +18 -4
- data/docs/formats.md +644 -16
- data/docs/path.md +18 -2
- data/docs/streams.md +54 -9
- data/docs/tutorial.md +14 -2
- data/docs/upgrading.md +470 -23
- data/lib/io_streams/builder/deprecations.rb +250 -0
- data/lib/io_streams/builder.rb +256 -39
- data/lib/io_streams/bzip2/reader.rb +18 -2
- data/lib/io_streams/bzip2/writer.rb +11 -2
- data/lib/io_streams/deprecation/methods.rb +51 -0
- data/lib/io_streams/deprecation/once.rb +27 -0
- data/lib/io_streams/deprecation/option_names.rb +26 -0
- data/lib/io_streams/deprecation/words.rb +42 -0
- data/lib/io_streams/deprecation.rb +259 -0
- data/lib/io_streams/encode/reader.rb +54 -3
- data/lib/io_streams/encode/writer.rb +19 -2
- data/lib/io_streams/encode.rb +99 -0
- data/lib/io_streams/errors.rb +51 -0
- data/lib/io_streams/gzip/reader.rb +11 -0
- data/lib/io_streams/io_streams.rb +365 -35
- data/lib/io_streams/line/reader.rb +2 -1
- data/lib/io_streams/line/utf8_reader.rb +23 -0
- data/lib/io_streams/path.rb +125 -35
- data/lib/io_streams/paths/file.rb +72 -9
- data/lib/io_streams/paths/http.rb +48 -2
- data/lib/io_streams/paths/matcher.rb +6 -3
- data/lib/io_streams/paths/s3.rb +180 -9
- data/lib/io_streams/paths/sftp.rb +47 -3
- data/lib/io_streams/paths.rb +23 -0
- data/lib/io_streams/pgp/reader.rb +18 -0
- data/lib/io_streams/pgp/writer.rb +16 -0
- data/lib/io_streams/pgp.rb +16 -2
- data/lib/io_streams/reader.rb +44 -0
- data/lib/io_streams/record/reader.rb +59 -68
- data/lib/io_streams/record/writer.rb +17 -10
- data/lib/io_streams/row/header_row.rb +21 -0
- data/lib/io_streams/row/reader.rb +52 -15
- data/lib/io_streams/row/writer.rb +64 -13
- data/lib/io_streams/schema/columns.rb +456 -0
- data/lib/io_streams/schema/constraints.rb +99 -0
- data/lib/io_streams/schema/field.rb +967 -0
- data/lib/io_streams/schema/json_number.rb +56 -0
- data/lib/io_streams/schema/keys.rb +78 -0
- data/lib/io_streams/schema/pattern.rb +257 -0
- data/lib/io_streams/schema/source.rb +31 -0
- data/lib/io_streams/schema.rb +616 -0
- data/lib/io_streams/stream.rb +836 -67
- data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
- data/lib/io_streams/tabular/header.rb +29 -0
- data/lib/io_streams/tabular/header_mapping.rb +332 -0
- data/lib/io_streams/tabular/mapping.rb +165 -0
- data/lib/io_streams/tabular/parser/array.rb +5 -0
- data/lib/io_streams/tabular/parser/base.rb +24 -0
- data/lib/io_streams/tabular/parser/fixed.rb +197 -0
- data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
- data/lib/io_streams/tabular/parser/hash.rb +10 -0
- data/lib/io_streams/tabular/parser/json.rb +17 -0
- data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
- data/lib/io_streams/tabular/schema_mapping.rb +361 -0
- data/lib/io_streams/tabular.rb +344 -30
- data/lib/io_streams/utils.rb +441 -6
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +27 -0
- data/lib/io_streams/xlsx/cells.rb +127 -0
- data/lib/io_streams/xlsx/reader.rb +56 -0
- data/lib/io_streams/xlsx/schema_reader.rb +90 -0
- data/lib/io_streams/xlsx/v3_writer.rb +16 -0
- data/lib/io_streams/zip/reader.rb +7 -0
- data/lib/io_streams/zip/writer.rb +24 -2
- data/lib/iostreams.rb +21 -17
- metadata +28 -2
data/lib/io_streams/utils.rb
CHANGED
|
@@ -5,6 +5,81 @@ module IOStreams
|
|
|
5
5
|
module Utils
|
|
6
6
|
MAX_TEMP_FILE_NAME_ATTEMPTS = 5
|
|
7
7
|
|
|
8
|
+
# Replaces the value of a sensitive option for display, see .redact_options.
|
|
9
|
+
FILTERED = "[FILTERED]".freeze
|
|
10
|
+
|
|
11
|
+
# Option names whose values are hidden as a precaution, even when they are not declared sensitive.
|
|
12
|
+
# Matched against the name in lower case without `_` or `-`, see .redact_options.
|
|
13
|
+
SENSITIVE_NAME = /passphrase|password|secret|token|credential|auth|apikey|privatekey/
|
|
14
|
+
private_constant :SENSITIVE_NAME
|
|
15
|
+
|
|
16
|
+
# Returns [Hash] the options with the value of each sensitive option replaced with FILTERED, so that they can be
|
|
17
|
+
# displayed, for example by `#inspect`, or in a web interface.
|
|
18
|
+
#
|
|
19
|
+
# An option is sensitive when its name is one of the supplied `sensitive_names`, or, as a precaution, contains
|
|
20
|
+
# `passphrase`, `password`, `secret`, `token`, `credential`, `auth`, `apikey` or `privatekey`. Names are compared in
|
|
21
|
+
# lower case, ignoring `_` and `-`, so that `:identity_key` matches the ssh option "IdentityKey", and the header
|
|
22
|
+
# "X-Api-Key" contains `apikey`. The options in a Hash value, such as `ssh_options:` or `client:`, are redacted the
|
|
23
|
+
# same way, as are those in an Array value, whether its elements are Hashes or `[name, value]` pairs.
|
|
24
|
+
def self.redact_options(options, sensitive_names = [])
|
|
25
|
+
redact_hash(options, sensitive_names.map { |name| normalize_option_name(name) })
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# Returns [Hash] the options of each stream, by its name, such as those supplied to `IOStreams::Stream#stream`, with
|
|
29
|
+
# the value of each sensitive option replaced with FILTERED, as the extension that the block returns for the name of
|
|
30
|
+
# the stream decides, see `IOStreams::Extension#redact_options`.
|
|
31
|
+
#
|
|
32
|
+
# Every value of a stream is replaced when the block returns nil, since no extension can say which are sensitive,
|
|
33
|
+
# and options that are not a Hash are replaced whole, since they cannot say which of their values are sensitive.
|
|
34
|
+
# Options that are nil are kept.
|
|
35
|
+
def self.redact_streams(streams)
|
|
36
|
+
streams.to_h do |name, options|
|
|
37
|
+
next [name, options] if options.nil?
|
|
38
|
+
next [name, FILTERED] unless options.is_a?(Hash)
|
|
39
|
+
|
|
40
|
+
extension = yield(name)
|
|
41
|
+
[name, extension ? extension.redact_options(options) : options.transform_values { FILTERED }]
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def self.redact_hash(options, sensitive)
|
|
46
|
+
options.to_h { |name, value| [name, redact_option(name, value, sensitive)] }
|
|
47
|
+
end
|
|
48
|
+
private_class_method :redact_hash
|
|
49
|
+
|
|
50
|
+
# Returns the value of the named option for display, see .redact_options.
|
|
51
|
+
def self.redact_option(name, value, sensitive)
|
|
52
|
+
key = normalize_option_name(name)
|
|
53
|
+
return FILTERED if sensitive.include?(key) || key.match?(SENSITIVE_NAME)
|
|
54
|
+
|
|
55
|
+
redact_value(value, sensitive)
|
|
56
|
+
end
|
|
57
|
+
private_class_method :redact_option
|
|
58
|
+
|
|
59
|
+
# Returns the value with the options within it redacted, when it is a Hash or an Array, see .redact_options.
|
|
60
|
+
def self.redact_value(value, sensitive)
|
|
61
|
+
case value
|
|
62
|
+
when Hash
|
|
63
|
+
redact_hash(value, sensitive)
|
|
64
|
+
when Array
|
|
65
|
+
value.map do |element|
|
|
66
|
+
if element.is_a?(Array) && element.size == 2 && (element.first.is_a?(String) || element.first.is_a?(Symbol))
|
|
67
|
+
[element.first, redact_option(element.first, element.last, sensitive)]
|
|
68
|
+
else
|
|
69
|
+
redact_value(element, sensitive)
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
else
|
|
73
|
+
value
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
private_class_method :redact_value
|
|
77
|
+
|
|
78
|
+
def self.normalize_option_name(name)
|
|
79
|
+
name.to_s.downcase.delete("_-")
|
|
80
|
+
end
|
|
81
|
+
private_class_method :normalize_option_name
|
|
82
|
+
|
|
8
83
|
# Lazy load dependent gem so that it remains a soft dependency.
|
|
9
84
|
def self.load_soft_dependency(gem_name, stream_type, require_name = gem_name)
|
|
10
85
|
require require_name
|
|
@@ -12,17 +87,47 @@ module IOStreams
|
|
|
12
87
|
raise(LoadError, "Please install the gem '#{gem_name}' to support #{stream_type}. #{e.message}")
|
|
13
88
|
end
|
|
14
89
|
|
|
15
|
-
# Log a warning for options that a stream ignores
|
|
16
|
-
|
|
90
|
+
# Log a warning for options that a stream ignores when reading (type: :reader) or writing (:writer): those that the
|
|
91
|
+
# class of the other direction takes, other_names, which v3.0 also accepts, and ignores, and the others, which v3.0
|
|
92
|
+
# raises `ArgumentError` for. A path supplies other_names, and the stream as it was set, see
|
|
93
|
+
# `IOStreams::Builder#taken_options`; without them, as when a class is opened directly, each option is one that
|
|
94
|
+
# v3.0 raises for, since its class takes only its own options.
|
|
95
|
+
#
|
|
96
|
+
# Those of neither direction are deprecated, and report a deprecation warning when they are set, see
|
|
97
|
+
# `IOStreams::Builder#option`.
|
|
98
|
+
def self.warn_unknown_options(unknown, stream, type, valid, other_names = [])
|
|
17
99
|
return if unknown.empty?
|
|
18
100
|
|
|
19
|
-
|
|
101
|
+
other = unknown.keys & other_names
|
|
102
|
+
neither = unknown.keys - other
|
|
103
|
+
direction = type == :reader ? "reading" : "writing"
|
|
104
|
+
unless other.empty?
|
|
105
|
+
IOStreams.logger&.warn(
|
|
106
|
+
"Ignoring #{other.map(&:inspect).join(', ')} when #{direction} a #{stream.inspect} stream, since " \
|
|
107
|
+
"#{other.size == 1 ? 'it only applies' : 'they only apply'} when #{type == :reader ? 'writing' : 'reading'}."
|
|
108
|
+
)
|
|
109
|
+
end
|
|
110
|
+
return if neither.empty?
|
|
111
|
+
|
|
20
112
|
IOStreams.logger&.warn(
|
|
21
|
-
"Ignoring unknown #{
|
|
22
|
-
"stream. Valid options: #{valid.map(&:inspect).join(', ')}.
|
|
113
|
+
"Ignoring unknown #{neither.size == 1 ? 'option' : 'options'} #{neither.map(&:inspect).join(', ')} when " \
|
|
114
|
+
"#{direction} a #{stream.inspect} stream. Valid options: #{valid.map(&:inspect).join(', ')}. " \
|
|
115
|
+
"In v3.0 this will raise ArgumentError."
|
|
23
116
|
)
|
|
24
117
|
end
|
|
25
118
|
|
|
119
|
+
# Returns [String] the string as UTF-8 text that can be displayed or logged, such as the name of a file, which is not
|
|
120
|
+
# always valid UTF-8, since a file system stores the bytes that a program gave it: a binary string is read as UTF-8,
|
|
121
|
+
# and each byte that is not valid UTF-8 is shown as `\xHH`.
|
|
122
|
+
#
|
|
123
|
+
# For example the name of a file in Latin-1, `caf\xE9.csv`, is displayed as "caf\\xE9.csv".
|
|
124
|
+
def self.display_text(string)
|
|
125
|
+
text = string.encoding == Encoding::BINARY ? string.dup.force_encoding(Encoding::UTF_8) : string
|
|
126
|
+
return text if text.valid_encoding?
|
|
127
|
+
|
|
128
|
+
text.scrub { |bytes| bytes.unpack("C*").map { |byte| format("\\x%02X", byte) }.join }
|
|
129
|
+
end
|
|
130
|
+
|
|
26
131
|
# Helper method: Returns [true|false] if a value is blank?
|
|
27
132
|
def self.blank?(value)
|
|
28
133
|
return true if value.nil?
|
|
@@ -31,6 +136,106 @@ module IOStreams
|
|
|
31
136
|
value.respond_to?(:empty?) ? value.empty? : !value
|
|
32
137
|
end
|
|
33
138
|
|
|
139
|
+
# Text that is blank, see .blank_text?: empty, or only whitespace, as Rails' `String#blank?` matches it.
|
|
140
|
+
BLANK_TEXT = /\A[[:space:]]*\z/
|
|
141
|
+
private_constant :BLANK_TEXT
|
|
142
|
+
|
|
143
|
+
# The pattern of BLANK_TEXT in each encoding that it cannot be matched against, such as UTF-16, see .blank_text?.
|
|
144
|
+
@encoded_blank_texts = {}
|
|
145
|
+
@encoded_blank_texts_mutex = Mutex.new
|
|
146
|
+
|
|
147
|
+
# The rules below of what is blank apply to the rows and records read and written with a schema, see
|
|
148
|
+
# `IOStreams::Tabular::SchemaMapping`, as IOStreams 3.0 applies them to every file, while a file without a schema
|
|
149
|
+
# keeps those of .blank?.
|
|
150
|
+
|
|
151
|
+
# Returns [true|false] whether the text is blank, by the rules of Rails' `String#blank?`: empty, or only whitespace,
|
|
152
|
+
# which is any Unicode whitespace, such as a no-break space (U+00A0) or an ideographic space (U+3000), not only a
|
|
153
|
+
# space, tab, or line ending. Text in an encoding that is not ASCII compatible, such as UTF-16, is matched in its own
|
|
154
|
+
# encoding, as Rails matches it.
|
|
155
|
+
#
|
|
156
|
+
# Raises ArgumentError for text whose bytes are not valid in its encoding, as Rails does.
|
|
157
|
+
def self.blank_text?(text)
|
|
158
|
+
text.empty? || BLANK_TEXT.match?(text)
|
|
159
|
+
rescue Encoding::CompatibilityError
|
|
160
|
+
encoded_blank_text(text.encoding).match?(text)
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Returns [true|false] whether a value within a row or record is blank, so that it holds no data: nil, or text that
|
|
164
|
+
# is blank, see .blank_text?. Any other value is data, such as false, 0, an empty Array, or text whose bytes are not
|
|
165
|
+
# valid in its encoding, since it is not text that is known to be blank.
|
|
166
|
+
def self.blank_value?(value)
|
|
167
|
+
value.nil? || (value.is_a?(String) && value.valid_encoding? && blank_text?(value))
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# Returns [true|false] whether a row or record is blank, so that it holds no data: nil, or an Array or Hash without
|
|
171
|
+
# values, or whose every value is blank, see .blank_value?, such as the row of a line of CSV of only commas, or a
|
|
172
|
+
# record of JSON whose every value is null.
|
|
173
|
+
def self.blank_row?(row)
|
|
174
|
+
case row
|
|
175
|
+
when nil then true
|
|
176
|
+
when ::Hash then row.each_value.all? { |value| blank_value?(value) }
|
|
177
|
+
when ::Array then row.all? { |value| blank_value?(value) }
|
|
178
|
+
else false
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# Returns [Regexp] the pattern of blank text in the supplied encoding, see .blank_text?, created once for each one.
|
|
183
|
+
def self.encoded_blank_text(encoding)
|
|
184
|
+
@encoded_blank_texts_mutex.synchronize do
|
|
185
|
+
@encoded_blank_texts[encoding] ||=
|
|
186
|
+
Regexp.new(BLANK_TEXT.source.encode(encoding), BLANK_TEXT.options | Regexp::FIXEDENCODING)
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
private_class_method :encoded_blank_text
|
|
190
|
+
|
|
191
|
+
# Returns [String] the text, taken as UTF-8 when it is binary, as IOStreams 2 reads a file without an encoding,
|
|
192
|
+
# see `IOStreams::Stream#encoding`, such as a file read with a schema, whose text is UTF-8, see
|
|
193
|
+
# `IOStreams::Stream#schema`. Text in any other encoding is returned as it is.
|
|
194
|
+
#
|
|
195
|
+
# Raises `IOStreams::Errors::InvalidEncoding` when it is not valid UTF-8.
|
|
196
|
+
def self.utf8(text)
|
|
197
|
+
return text unless text.encoding == ::Encoding::BINARY
|
|
198
|
+
|
|
199
|
+
utf8 = text.dup.force_encoding(::Encoding::UTF_8)
|
|
200
|
+
return utf8 if utf8.valid_encoding?
|
|
201
|
+
|
|
202
|
+
raise(invalid_utf8(utf8))
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
# Returns [IOStreams::Errors::InvalidEncoding] the error for text taken as UTF-8 that is not valid UTF-8, naming
|
|
206
|
+
# its first invalid character, and the line that it is on when the number of the line that the text starts on is
|
|
207
|
+
# supplied, counting the line breaks within the text, such as within a quoted CSV value.
|
|
208
|
+
def self.invalid_utf8(utf8, line_number: nil)
|
|
209
|
+
invalid = nil
|
|
210
|
+
utf8.each_char do |char|
|
|
211
|
+
unless char.valid_encoding?
|
|
212
|
+
invalid = char.b
|
|
213
|
+
break
|
|
214
|
+
end
|
|
215
|
+
line_number += 1 if line_number && char == "\n"
|
|
216
|
+
end
|
|
217
|
+
on_line = line_number ? " on line #{line_number}" : ""
|
|
218
|
+
IOStreams::Errors::InvalidEncoding.new(
|
|
219
|
+
"#{invalid.inspect} is not valid UTF-8#{on_line}. Set the encoding of the file with #encoding",
|
|
220
|
+
line_number: line_number,
|
|
221
|
+
error_char: invalid,
|
|
222
|
+
source_encoding: ::Encoding::BINARY,
|
|
223
|
+
destination_encoding: ::Encoding::UTF_8
|
|
224
|
+
)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Returns a frozen copy of the value, whose Hashes, Arrays and Strings are frozen copies too, so that holding a
|
|
228
|
+
# setting, such as a schema, does not freeze the objects that the caller supplied, and the caller changing them
|
|
229
|
+
# does not change the setting. Any other value, such as a Symbol, is returned as it is.
|
|
230
|
+
def self.frozen_copy(value)
|
|
231
|
+
case value
|
|
232
|
+
when Hash then value.to_h { |key, item| [frozen_copy(key), frozen_copy(item)] }.freeze
|
|
233
|
+
when Array then value.map { |item| frozen_copy(item) }.freeze
|
|
234
|
+
when String then value.frozen? ? value : value.dup.freeze
|
|
235
|
+
else value
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
34
239
|
# Yields the path to a temporary file_name.
|
|
35
240
|
#
|
|
36
241
|
# File is deleted upon completion if present.
|
|
@@ -64,8 +269,237 @@ module IOStreams
|
|
|
64
269
|
end
|
|
65
270
|
end
|
|
66
271
|
|
|
272
|
+
module PathName
|
|
273
|
+
# The byte of the separator of the names in a path, `/`, which every encoding that a path can be in, those that are
|
|
274
|
+
# ASCII-compatible, holds as this byte, see .join.
|
|
275
|
+
SEPARATOR = "/".ord
|
|
276
|
+
private_constant :SEPARATOR
|
|
277
|
+
|
|
278
|
+
# The byte of the `.` that starts the extension of a name, see .extname.
|
|
279
|
+
DOT = ".".ord
|
|
280
|
+
private_constant :DOT
|
|
281
|
+
|
|
282
|
+
# Whether `::File.join`, `::File.dirname`, `::File.extname` and `::File.basename` keep the bytes of a name that is
|
|
283
|
+
# not valid UTF-8, as CRuby does, rather than changing them, as JRuby does, see
|
|
284
|
+
# https://github.com/jruby/jruby/issues/9880, such as the name of a file in Latin-1 that SFTP lists as bytes, or
|
|
285
|
+
# that is labelled UTF-8, so that the path no longer names the file. Found by what they do, so that
|
|
286
|
+
# IOStreams uses them again once JRuby keeps the bytes.
|
|
287
|
+
FILE_KEEPS_NAMES =
|
|
288
|
+
begin
|
|
289
|
+
name = "a\xE9".b
|
|
290
|
+
label = name.dup.force_encoding(Encoding::UTF_8)
|
|
291
|
+
::File.join(name, name).b == "a\xE9/a\xE9".b && ::File.dirname("#{name}/x").b == name &&
|
|
292
|
+
::File.extname("x.".b + name).b == ".a\xE9".b && ::File.basename("x/#{label}").b == name
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# Returns [String] the supplied names joined into a path, as `::File.join` joins Strings: with one `/` between
|
|
296
|
+
# each name and the next, whose separators decide which are kept, keeping the bytes and the encoding of each name,
|
|
297
|
+
# where `::File.join` does, see FILE_KEEPS_NAMES, otherwise by .join_names.
|
|
298
|
+
def self.join(*names)
|
|
299
|
+
FILE_KEEPS_NAMES ? ::File.join(*names) : join_names(names)
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
# Returns [String] the directory of the supplied path, as `::File.dirname` returns it, see .join.
|
|
303
|
+
def self.dirname(path)
|
|
304
|
+
FILE_KEEPS_NAMES ? ::File.dirname(path) : dirname_of(path)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Returns [String] the extension of the last name of the supplied path, as `::File.extname` returns it, see .join.
|
|
308
|
+
def self.extname(path)
|
|
309
|
+
FILE_KEEPS_NAMES ? ::File.extname(path) : extname_of(path)
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Returns [String] the last name of the supplied path, as `::File.basename` returns it, without the supplied
|
|
313
|
+
# suffix when it ends with it, see .join.
|
|
314
|
+
def self.basename(path, suffix = nil)
|
|
315
|
+
return basename_of(path, suffix) unless FILE_KEEPS_NAMES
|
|
316
|
+
|
|
317
|
+
suffix.nil? ? ::File.basename(path) : ::File.basename(path, suffix)
|
|
318
|
+
end
|
|
319
|
+
|
|
320
|
+
# Returns [String] the supplied Strings joined into a path, as CRuby's `File.join` joins them on POSIX, by their
|
|
321
|
+
# bytes: each name after the first is joined to the path so far with one `/`, unless the path ends with `/`, when
|
|
322
|
+
# it is added as it is, or the name starts with `/`, when the path loses its trailing `/` before it. The path starts
|
|
323
|
+
# in the encoding of the first name, and takes that of a name whose text needs it, as for `String#<<`, which raises
|
|
324
|
+
# `Encoding::CompatibilityError` for names that have no encoding in common.
|
|
325
|
+
#
|
|
326
|
+
# Used where `::File.join` changes names, see .join, and public, so that its tests compare it with CRuby's.
|
|
327
|
+
def self.join_names(names)
|
|
328
|
+
joined = String.new(encoding: names.first&.encoding || Encoding::BINARY)
|
|
329
|
+
names.each_with_index do |name, index|
|
|
330
|
+
if index.positive?
|
|
331
|
+
tail = trailing_separators(joined)
|
|
332
|
+
if name.getbyte(0) == SEPARATOR
|
|
333
|
+
joined = joined.byteslice(0, tail)
|
|
334
|
+
elsif tail == joined.bytesize
|
|
335
|
+
joined << "/"
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
joined << name
|
|
339
|
+
end
|
|
340
|
+
joined
|
|
341
|
+
end
|
|
342
|
+
|
|
343
|
+
# Returns [String] the directory of the supplied path, as CRuby's `File.dirname` returns it on POSIX, by its bytes:
|
|
344
|
+
# the path before the separators of its last name, `/` for a path of only separators, or `.` for a single name, in
|
|
345
|
+
# the encoding of the path.
|
|
346
|
+
#
|
|
347
|
+
# Used where `::File.dirname` changes names, see .dirname, and public, so that its tests compare it with CRuby's.
|
|
348
|
+
def self.dirname_of(path)
|
|
349
|
+
root = leading_separators(path)
|
|
350
|
+
start = root > 1 ? root - 1 : 0
|
|
351
|
+
finish = last_separator(path, root) || root
|
|
352
|
+
return String.new(".", encoding: path.encoding) if finish == start
|
|
353
|
+
|
|
354
|
+
path.byteslice(start, finish - start)
|
|
355
|
+
end
|
|
356
|
+
|
|
357
|
+
# Returns [String] the extension of the last name of the supplied path, as CRuby's `File.extname` returns it on
|
|
358
|
+
# POSIX, by its bytes: from its last `.`, unless only dots start the name, in the encoding of the path, or an
|
|
359
|
+
# empty String when it has none.
|
|
360
|
+
#
|
|
361
|
+
# Used where `::File.extname` changes names, see .extname, and public, so that its tests compare it with CRuby's.
|
|
362
|
+
def self.extname_of(path)
|
|
363
|
+
separator = last_separator(path, 0)
|
|
364
|
+
position = separator ? separator + 1 : 0
|
|
365
|
+
position += 1 while position < path.bytesize && path.getbyte(position) == SEPARATOR
|
|
366
|
+
position += 1 while position < path.bytesize && path.getbyte(position) == DOT
|
|
367
|
+
dot = nil
|
|
368
|
+
while position < path.bytesize
|
|
369
|
+
byte = path.getbyte(position)
|
|
370
|
+
break if byte == SEPARATOR
|
|
371
|
+
|
|
372
|
+
dot = position if byte == DOT
|
|
373
|
+
position += 1
|
|
374
|
+
end
|
|
375
|
+
return "".b if dot.nil?
|
|
376
|
+
|
|
377
|
+
path.byteslice(dot, position - dot)
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# Returns [String] the last name of the supplied path, as CRuby's `File.basename` returns it on POSIX, by its
|
|
381
|
+
# bytes: the name after its last separators, without the separators at its end, `/` for a path of only
|
|
382
|
+
# separators, in the encoding of the path. The supplied suffix is removed when the name ends with it, `.*` removes
|
|
383
|
+
# the extension of the name, see .extname, and a character followed by `*` removes the name from the last of that
|
|
384
|
+
# character, unless that would leave nothing. A suffix without an encoding in common with the path is not
|
|
385
|
+
# removed.
|
|
386
|
+
#
|
|
387
|
+
# Raises ArgumentError, as CRuby does, when the first character of the suffix, or for a character followed by
|
|
388
|
+
# `*`, a character of the name, is not valid in the encoding that they have in common.
|
|
389
|
+
#
|
|
390
|
+
# Used where `::File.basename` changes names, see .basename, and public, so that its tests compare it with CRuby's.
|
|
391
|
+
def self.basename_of(path, suffix = nil)
|
|
392
|
+
return path.dup if path.empty?
|
|
393
|
+
|
|
394
|
+
start = leading_separators(path)
|
|
395
|
+
return path.byteslice(start - 1, 1) if start == path.bytesize
|
|
396
|
+
|
|
397
|
+
separator = last_separator(path, start)
|
|
398
|
+
if separator
|
|
399
|
+
start = separator
|
|
400
|
+
start += 1 while path.getbyte(start) == SEPARATOR
|
|
401
|
+
end
|
|
402
|
+
length = trailing_separators(path) - start
|
|
403
|
+
encoding = suffix && Encoding.compatible?(path, suffix)
|
|
404
|
+
kept = encoding ? without_suffix(path, start, length, suffix.dup.force_encoding(encoding)) : 0
|
|
405
|
+
path.byteslice(start, kept.positive? ? kept : length)
|
|
406
|
+
end
|
|
407
|
+
|
|
408
|
+
# Returns [Integer] the number of bytes of the name of the supplied length, from the supplied position of the
|
|
409
|
+
# supplied path, that are kept once the supplied suffix is removed, see .basename_of, or 0 when the suffix is not
|
|
410
|
+
# removed, as CRuby's `rmext` does.
|
|
411
|
+
def self.without_suffix(path, start, length, suffix)
|
|
412
|
+
return 0 if suffix.empty?
|
|
413
|
+
|
|
414
|
+
first = valid_character(suffix[0])
|
|
415
|
+
if suffix.bytesize == first.bytesize + 1 && suffix.getbyte(first.bytesize) == "*".ord
|
|
416
|
+
return stem_length(path, start, length) if first == "."
|
|
417
|
+
|
|
418
|
+
name = valid_character(path.byteslice(start, length).force_encoding(suffix.encoding))
|
|
419
|
+
index = name.rindex(first)
|
|
420
|
+
return index ? name[0, index].bytesize : length
|
|
421
|
+
end
|
|
422
|
+
return length if length < suffix.bytesize
|
|
423
|
+
|
|
424
|
+
from = start + length - suffix.bytesize
|
|
425
|
+
return 0 unless character_start?(path, from, suffix.encoding)
|
|
426
|
+
|
|
427
|
+
path.byteslice(from, suffix.bytesize).b == suffix.b ? length - suffix.bytesize : 0
|
|
428
|
+
end
|
|
429
|
+
|
|
430
|
+
# Returns [String] the supplied text.
|
|
431
|
+
#
|
|
432
|
+
# Raises ArgumentError when it is not valid in its encoding, as CRuby does when it reads its characters.
|
|
433
|
+
def self.valid_character(text)
|
|
434
|
+
return text if text.valid_encoding?
|
|
435
|
+
|
|
436
|
+
raise(ArgumentError, "invalid byte sequence in #{text.encoding}")
|
|
437
|
+
end
|
|
438
|
+
private_class_method :valid_character
|
|
439
|
+
private_class_method :without_suffix
|
|
440
|
+
|
|
441
|
+
# Returns [Integer] the number of bytes of the name of the supplied length, from the supplied position of the
|
|
442
|
+
# supplied path, before its extension, see .extname_of: before its last `.`, unless only dots start the name.
|
|
443
|
+
def self.stem_length(path, start, length)
|
|
444
|
+
position = start
|
|
445
|
+
position += 1 while position - start < length && path.getbyte(position) == DOT
|
|
446
|
+
dot = nil
|
|
447
|
+
while position - start < length
|
|
448
|
+
dot = position if path.getbyte(position) == DOT
|
|
449
|
+
position += 1
|
|
450
|
+
end
|
|
451
|
+
dot ? dot - start : length
|
|
452
|
+
end
|
|
453
|
+
private_class_method :stem_length
|
|
454
|
+
|
|
455
|
+
# Returns [true|false] whether a character of the supplied path, read in the supplied encoding, starts at the
|
|
456
|
+
# supplied position: always for a single byte encoding, and for UTF-8 unless its byte continues a character.
|
|
457
|
+
def self.character_start?(path, position, encoding)
|
|
458
|
+
return true unless encoding == Encoding::UTF_8
|
|
459
|
+
|
|
460
|
+
!(0x80..0xBF).cover?(path.getbyte(position))
|
|
461
|
+
end
|
|
462
|
+
private_class_method :character_start?
|
|
463
|
+
|
|
464
|
+
# Returns [Integer] the number of bytes of the supplied path before the separators at its end.
|
|
465
|
+
def self.trailing_separators(path)
|
|
466
|
+
tail = path.bytesize
|
|
467
|
+
tail -= 1 while tail.positive? && path.getbyte(tail - 1) == SEPARATOR
|
|
468
|
+
tail
|
|
469
|
+
end
|
|
470
|
+
private_class_method :trailing_separators
|
|
471
|
+
|
|
472
|
+
# Returns [Integer] the number of separators at the start of the supplied path.
|
|
473
|
+
def self.leading_separators(path)
|
|
474
|
+
count = 0
|
|
475
|
+
count += 1 while count < path.bytesize && path.getbyte(count) == SEPARATOR
|
|
476
|
+
count
|
|
477
|
+
end
|
|
478
|
+
private_class_method :leading_separators
|
|
479
|
+
|
|
480
|
+
# Returns [Integer] the position of the first of the last separators of the supplied path, from the supplied
|
|
481
|
+
# position on, that a name follows, or nil when it has none, as CRuby's `strrdirsep` finds it.
|
|
482
|
+
def self.last_separator(path, from)
|
|
483
|
+
last = nil
|
|
484
|
+
position = from
|
|
485
|
+
while position < path.bytesize
|
|
486
|
+
if path.getbyte(position) == SEPARATOR
|
|
487
|
+
separator = position
|
|
488
|
+
position += 1 while position < path.bytesize && path.getbyte(position) == SEPARATOR
|
|
489
|
+
break if position >= path.bytesize
|
|
490
|
+
|
|
491
|
+
last = separator
|
|
492
|
+
else
|
|
493
|
+
position += 1
|
|
494
|
+
end
|
|
495
|
+
end
|
|
496
|
+
last
|
|
497
|
+
end
|
|
498
|
+
private_class_method :last_separator
|
|
499
|
+
end
|
|
500
|
+
|
|
67
501
|
class URI
|
|
68
|
-
attr_reader :scheme, :hostname, :path, :user, :password, :port, :query
|
|
502
|
+
attr_reader :scheme, :hostname, :path, :user, :password, :port, :query, :fragment
|
|
69
503
|
|
|
70
504
|
def initialize(url)
|
|
71
505
|
url = url.gsub(" ", "%20")
|
|
@@ -76,6 +510,7 @@ module IOStreams
|
|
|
76
510
|
@user = uri.user
|
|
77
511
|
@password = uri.password
|
|
78
512
|
@port = uri.port
|
|
513
|
+
@fragment = uri.fragment
|
|
79
514
|
return unless uri.query
|
|
80
515
|
|
|
81
516
|
@query = {}
|
data/lib/io_streams/version.rb
CHANGED
data/lib/io_streams/writer.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
module IOStreams
|
|
2
2
|
class Writer
|
|
3
|
+
# Answers the names of the options that this writer takes in IOStreams 3.0, and those that it replaces.
|
|
4
|
+
extend Deprecation::OptionNames
|
|
5
|
+
|
|
3
6
|
# Returns [Array<Symbol>] the names of the options this writer accepts,
|
|
4
7
|
# or [nil] when the writer does not declare them.
|
|
5
8
|
#
|
|
@@ -9,6 +12,30 @@ module IOStreams
|
|
|
9
12
|
nil
|
|
10
13
|
end
|
|
11
14
|
|
|
15
|
+
# Returns [Array<Symbol>] the names of the options whose values must not be displayed, such as a passphrase,
|
|
16
|
+
# so that `#inspect` on a path does not display them, see `IOStreams::Extension#redact_options`.
|
|
17
|
+
def self.sensitive_option_names
|
|
18
|
+
[]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# The methods of an IO that write, which every stream that v3.0 supplies to a writer block answers, see
|
|
22
|
+
# .v3_stream_methods.
|
|
23
|
+
V3_WRITE_METHODS = %i[write << print puts printf].freeze
|
|
24
|
+
|
|
25
|
+
# Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a writer block, in place of the one
|
|
26
|
+
# that this writer yields, answers, when it does not answer all of the methods of that one, otherwise nil, for a
|
|
27
|
+
# writer whose stream answers the same methods in both versions, see `IOStreams::Reader.v3_stream_methods`, whose
|
|
28
|
+
# parameters and block it takes too.
|
|
29
|
+
def self.v3_stream_methods(_options)
|
|
30
|
+
nil
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Returns [Hash<Symbol:String>|nil] what the application uses in place of each method of the stream that this writer
|
|
34
|
+
# yields that v3.0 removes, see `IOStreams::Reader.v3_stream_advice`.
|
|
35
|
+
def self.v3_stream_advice
|
|
36
|
+
nil
|
|
37
|
+
end
|
|
38
|
+
|
|
12
39
|
# When a Writer does not support streams, we copy the stream to a local temp file
|
|
13
40
|
# and then pass that filename in for this reader.
|
|
14
41
|
def self.stream(output_stream, **args, &block)
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
|
|
5
|
+
module IOStreams
|
|
6
|
+
module Xlsx
|
|
7
|
+
# The class of the number format of each cell of a worksheet, from Excel's own number formats, and the text of a cell
|
|
8
|
+
# of a date or a time, as IOStreams 3.0 types them, so that a spreadsheet read with a schema, see
|
|
9
|
+
# `IOStreams::Xlsx::SchemaReader`, holds the same dates and times in IOStreams 2.2 and 3.0, rather than as creek
|
|
10
|
+
# classifies them, which takes every custom format with a letter of a date or a time as a date and time, such as
|
|
11
|
+
# `yyyy-mm-dd`, or `0.00" days"`:
|
|
12
|
+
#
|
|
13
|
+
# Number format | Text of the cell
|
|
14
|
+
# a date | its date in ISO 8601, such as `2026-10-25`
|
|
15
|
+
# a date and time | the time of its clock in UTC, such as `2026-10-25T09:30:00Z`, since Excel has no zone
|
|
16
|
+
# a time of day | the time of its clock, such as `09:30:00`
|
|
17
|
+
module Cells
|
|
18
|
+
# The seconds of a day.
|
|
19
|
+
DAY = 86_400
|
|
20
|
+
|
|
21
|
+
# The first day of a time of day, as Active Record holds the value of a `time` column.
|
|
22
|
+
TIME_OF_DAY = [2000, 1, 1].freeze
|
|
23
|
+
|
|
24
|
+
# The number of the day after the last date that Excel holds, 9999-12-31, since 1899-12-30.
|
|
25
|
+
LAST_DAY = 2_958_466
|
|
26
|
+
|
|
27
|
+
# The built-in number formats of Excel of dates, times of day, and dates with times, by their ids.
|
|
28
|
+
BUILT_IN_FORMATS = {
|
|
29
|
+
14 => :date, 15 => :date, 16 => :date, 17 => :date, 18 => :time, 19 => :time, 20 => :time, 21 => :time,
|
|
30
|
+
22 => :date_time, 45 => :time, 46 => :time, 47 => :time
|
|
31
|
+
}.freeze
|
|
32
|
+
|
|
33
|
+
# The parts of a custom format code that are not its tokens: quoted text, an escaped character, and a section in
|
|
34
|
+
# brackets other than an elapsed time, such as a color or a locale. A section never holds another `[`, so a match
|
|
35
|
+
# stops there, rather than scan the rest of the code again from each `[` of a workbook's code that never closes.
|
|
36
|
+
LITERALS = /"[^"]*"|\\.|_.|\*.|\[(?![hms]+\])[^\[\]]*\]/i
|
|
37
|
+
|
|
38
|
+
# Returns [Array<Symbol|nil>] the class of the number format of each style of a workbook, by the index of the style,
|
|
39
|
+
# as a cell names it with its `s` attribute: `:date`, `:time` or `:date_time`, or nil for a number, from Excel's own
|
|
40
|
+
# number formats, see BUILT_IN_FORMATS and .custom_format, rather than as creek finds them.
|
|
41
|
+
#
|
|
42
|
+
# Parameters:
|
|
43
|
+
# styles: [Nokogiri::XML::Document|nil]
|
|
44
|
+
# The styles of the workbook, `xl/styles.xml`, or nil for a workbook without them.
|
|
45
|
+
def self.number_formats(styles)
|
|
46
|
+
return [] if styles.nil?
|
|
47
|
+
|
|
48
|
+
custom = styles.css("styleSheet numFmts numFmt").to_h do |format|
|
|
49
|
+
[format["numFmtId"].to_i, custom_format(format["formatCode"].to_s)]
|
|
50
|
+
end
|
|
51
|
+
styles.css("styleSheet cellXfs xf").map do |style|
|
|
52
|
+
id = style["numFmtId"]&.to_i
|
|
53
|
+
BUILT_IN_FORMATS.fetch(id) { custom[id] } if id
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
# Returns [Symbol|nil] the class of a custom format code, by the first of its sections: `:date` with only a year,
|
|
58
|
+
# month or day, `:time` with only hours, minutes or seconds, `:date_time` with both, otherwise nil, for a number.
|
|
59
|
+
# An `m` is minutes after hours or before seconds, otherwise the month.
|
|
60
|
+
def self.custom_format(code)
|
|
61
|
+
tokens = code.split(";").first.to_s.gsub(LITERALS, "")
|
|
62
|
+
time = tokens.match?(%r{[hs]|am/pm|a/p}i)
|
|
63
|
+
date = tokens.match?(/[yd]/i) || (!time && tokens.match?(/m/i))
|
|
64
|
+
if date && time then :date_time
|
|
65
|
+
elsif date then :date
|
|
66
|
+
elsif time then :time
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
# Returns [String|nil] the text of a cell of a number type, the text of its `<v>` element, by the class of its
|
|
71
|
+
# number format, see .number_formats, with the first day of the workbook, see `Creek::Sheet#converter_options`, as
|
|
72
|
+
# IOStreams 3.0 types it, so that a field of its type reads it:
|
|
73
|
+
# a date in ISO 8601, such as "2026-10-25"
|
|
74
|
+
# a date and time its time in UTC, such as "2026-10-25T09:30:00Z", with its milliseconds when it has any
|
|
75
|
+
# a time the time of its clock, such as "09:30:00", or of a number of a day or more, its date
|
|
76
|
+
# and time, as a date and time
|
|
77
|
+
#
|
|
78
|
+
# Returns nil for a number of any other format, or one of a date or time format that is not a date that Excel
|
|
79
|
+
# holds, such as a negative number, which Excel shows as `####`, which is a number.
|
|
80
|
+
def self.date_text(text, style, base_date)
|
|
81
|
+
days = days(text) if style
|
|
82
|
+
return unless days
|
|
83
|
+
|
|
84
|
+
case style
|
|
85
|
+
when :date then (base_date + days.floor).iso8601
|
|
86
|
+
when :date_time then Schema::Field.any_text(date_time(days, base_date))
|
|
87
|
+
when :time then days < 1 ? time_of_day_text(time_of_day(days)) : Schema::Field.any_text(date_time(days, base_date))
|
|
88
|
+
end
|
|
89
|
+
rescue ArgumentError, ZeroDivisionError
|
|
90
|
+
nil
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Returns [Rational|nil] the days since the first day of the workbook of the text of a number, or nil for one
|
|
94
|
+
# outside the dates that Excel holds, from its first day up to 9999-12-31, such as a negative number, or one of a
|
|
95
|
+
# huge exponent, which is checked as a Float before it becomes a Rational, so that its cost is bounded.
|
|
96
|
+
def self.days(text)
|
|
97
|
+
days = Float(text)
|
|
98
|
+
Rational(text) if days >= 0 && days < LAST_DAY
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
# Returns [String] the time of the clock of a time of day, such as "09:30:00", with the milliseconds that Excel
|
|
102
|
+
# holds when it has any, such as "09:30:00.500", as a `time` field of IOStreams 3.0 writes it.
|
|
103
|
+
def self.time_of_day_text(time)
|
|
104
|
+
time.nsec.zero? ? time.strftime("%H:%M:%S") : time.strftime("%H:%M:%S.%3N")
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# Returns [Time] the time, in UTC, of the clock of the days since the first day of the workbook, to the millisecond
|
|
108
|
+
# that Excel holds.
|
|
109
|
+
def self.date_time(days, base_date)
|
|
110
|
+
date = base_date + days.floor
|
|
111
|
+
Time.utc(date.year, date.month, date.day) + seconds(days)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Returns [Time] the time of day of the fraction of a day, on 2000-01-01 in UTC, also when the fraction rounds to
|
|
115
|
+
# a whole day, which is midnight.
|
|
116
|
+
def self.time_of_day(days)
|
|
117
|
+
Time.utc(*TIME_OF_DAY) + (seconds(days) % DAY)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Returns [Rational] the seconds of the fraction of the day, to the millisecond.
|
|
121
|
+
def self.seconds(days)
|
|
122
|
+
((days - days.floor) * DAY).round(3)
|
|
123
|
+
end
|
|
124
|
+
private_class_method :days, :time_of_day_text, :date_time, :time_of_day, :seconds
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|