iostreams 2.1.0 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/docs/config.md +13 -9
- data/docs/copy_files.md +20 -2
- data/docs/extensions.md +18 -4
- data/docs/formats.md +644 -16
- data/docs/path.md +18 -2
- data/docs/streams.md +54 -9
- data/docs/tutorial.md +14 -2
- data/docs/upgrading.md +470 -23
- data/lib/io_streams/builder/deprecations.rb +250 -0
- data/lib/io_streams/builder.rb +256 -39
- data/lib/io_streams/bzip2/reader.rb +18 -2
- data/lib/io_streams/bzip2/writer.rb +11 -2
- data/lib/io_streams/deprecation/methods.rb +51 -0
- data/lib/io_streams/deprecation/once.rb +27 -0
- data/lib/io_streams/deprecation/option_names.rb +26 -0
- data/lib/io_streams/deprecation/words.rb +42 -0
- data/lib/io_streams/deprecation.rb +259 -0
- data/lib/io_streams/encode/reader.rb +54 -3
- data/lib/io_streams/encode/writer.rb +19 -2
- data/lib/io_streams/encode.rb +99 -0
- data/lib/io_streams/errors.rb +51 -0
- data/lib/io_streams/gzip/reader.rb +11 -0
- data/lib/io_streams/io_streams.rb +365 -35
- data/lib/io_streams/line/reader.rb +2 -1
- data/lib/io_streams/line/utf8_reader.rb +23 -0
- data/lib/io_streams/path.rb +125 -35
- data/lib/io_streams/paths/file.rb +72 -9
- data/lib/io_streams/paths/http.rb +48 -2
- data/lib/io_streams/paths/matcher.rb +6 -3
- data/lib/io_streams/paths/s3.rb +180 -9
- data/lib/io_streams/paths/sftp.rb +47 -3
- data/lib/io_streams/paths.rb +23 -0
- data/lib/io_streams/pgp/reader.rb +18 -0
- data/lib/io_streams/pgp/writer.rb +16 -0
- data/lib/io_streams/pgp.rb +16 -2
- data/lib/io_streams/reader.rb +44 -0
- data/lib/io_streams/record/reader.rb +59 -68
- data/lib/io_streams/record/writer.rb +17 -10
- data/lib/io_streams/row/header_row.rb +21 -0
- data/lib/io_streams/row/reader.rb +52 -15
- data/lib/io_streams/row/writer.rb +64 -13
- data/lib/io_streams/schema/columns.rb +456 -0
- data/lib/io_streams/schema/constraints.rb +99 -0
- data/lib/io_streams/schema/field.rb +967 -0
- data/lib/io_streams/schema/json_number.rb +56 -0
- data/lib/io_streams/schema/keys.rb +78 -0
- data/lib/io_streams/schema/pattern.rb +257 -0
- data/lib/io_streams/schema/source.rb +31 -0
- data/lib/io_streams/schema.rb +616 -0
- data/lib/io_streams/stream.rb +836 -67
- data/lib/io_streams/symmetric_encryption/reader.rb +8 -0
- data/lib/io_streams/tabular/header.rb +29 -0
- data/lib/io_streams/tabular/header_mapping.rb +332 -0
- data/lib/io_streams/tabular/mapping.rb +165 -0
- data/lib/io_streams/tabular/parser/array.rb +5 -0
- data/lib/io_streams/tabular/parser/base.rb +24 -0
- data/lib/io_streams/tabular/parser/fixed.rb +197 -0
- data/lib/io_streams/tabular/parser/fixed_fields.rb +160 -0
- data/lib/io_streams/tabular/parser/hash.rb +10 -0
- data/lib/io_streams/tabular/parser/json.rb +17 -0
- data/lib/io_streams/tabular/parser/json_fields.rb +132 -0
- data/lib/io_streams/tabular/schema_mapping.rb +361 -0
- data/lib/io_streams/tabular.rb +344 -30
- data/lib/io_streams/utils.rb +441 -6
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +27 -0
- data/lib/io_streams/xlsx/cells.rb +127 -0
- data/lib/io_streams/xlsx/reader.rb +56 -0
- data/lib/io_streams/xlsx/schema_reader.rb +90 -0
- data/lib/io_streams/xlsx/v3_writer.rb +16 -0
- data/lib/io_streams/zip/reader.rb +7 -0
- data/lib/io_streams/zip/writer.rb +24 -2
- data/lib/iostreams.rb +21 -17
- metadata +28 -2
data/lib/io_streams/paths/s3.rb
CHANGED
|
@@ -5,6 +5,10 @@ module IOStreams
|
|
|
5
5
|
class S3 < IOStreams::Path
|
|
6
6
|
attr_reader :bucket_name, :options
|
|
7
7
|
|
|
8
|
+
# The secret access key, also within `client:`, and the customer encryption keys, see
|
|
9
|
+
# IOStreams::Path.redact_options. A session token and credentials are always sensitive by name.
|
|
10
|
+
def self.sensitive_option_names = %i[secret_access_key sse_customer_key copy_source_sse_customer_key]
|
|
11
|
+
|
|
8
12
|
# Largest file size supported by the S3 copy object api.
|
|
9
13
|
S3_COPY_OBJECT_SIZE_LIMIT = 5 * 1024 * 1024 * 1024
|
|
10
14
|
|
|
@@ -178,14 +182,55 @@ module IOStreams
|
|
|
178
182
|
@client_options[:region] = region if region
|
|
179
183
|
end
|
|
180
184
|
|
|
181
|
-
@options = args
|
|
182
|
-
@options.merge!(uri.query.transform_keys(&:to_sym)) if uri.query
|
|
185
|
+
@options = uri.query ? args.merge(uri.query.transform_keys(&:to_sym)) : args
|
|
183
186
|
|
|
184
187
|
super(key)
|
|
188
|
+
# Each child that #each_child yields is a new path too, so nothing is checked while deprecation warnings are off.
|
|
189
|
+
return unless Warning[:deprecated]
|
|
190
|
+
|
|
191
|
+
warn_unusable_url(uri)
|
|
192
|
+
warn_client_arguments(client, access_key_id: access_key_id, secret_access_key: secret_access_key, region: region)
|
|
193
|
+
warn_unusable_options(args.keys) unless args.empty?
|
|
194
|
+
warn_unusable_query(uri.query) if uri.query
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# The S3 requests whose parameters IOStreams 3.0 takes as the options of a path, see .known_option_names.
|
|
198
|
+
OPTION_OPERATIONS = %i[get_object head_object put_object copy_object delete_object delete_objects list_objects_v2].freeze
|
|
199
|
+
private_constant :OPTION_OPERATIONS
|
|
200
|
+
|
|
201
|
+
# The parameters of the S3 requests that a path sets itself, such as `key`, which it overrides, and those of a
|
|
202
|
+
# listing that would change which keys it returns, or how, which IOStreams 3.0 does not take as the options of a
|
|
203
|
+
# path, see .known_option_names.
|
|
204
|
+
PATH_PARAMETERS = %i[
|
|
205
|
+
bucket key body response_target copy_source prefix continuation_token delimiter delete encoding_type start_after
|
|
206
|
+
].freeze
|
|
207
|
+
private_constant :PATH_PARAMETERS
|
|
208
|
+
|
|
209
|
+
# Returns [Array<Symbol>] the options of a path that IOStreams 3.0 takes, which raises ArgumentError for any other
|
|
210
|
+
# when the path is created: the parameters of its S3 requests, see OPTION_OPERATIONS, as the AWS SDK names them,
|
|
211
|
+
# other than those that the path sets itself, see PATH_PARAMETERS. Load the AWS SDK first.
|
|
212
|
+
def self.known_option_names
|
|
213
|
+
@known_option_names ||=
|
|
214
|
+
(OPTION_OPERATIONS.flat_map { |operation| ::Aws::S3::Client.api.operation(operation).input.shape.member_names }.uniq -
|
|
215
|
+
PATH_PARAMETERS).freeze
|
|
185
216
|
end
|
|
186
217
|
|
|
187
218
|
def to_s
|
|
188
|
-
::
|
|
219
|
+
Utils::PathName.join("s3://", bucket_name, path)
|
|
220
|
+
end
|
|
221
|
+
|
|
222
|
+
# Joining a name that starts with `/` to the bucket itself, see `IOStreams::Path#join`, gives the key that starts
|
|
223
|
+
# with `/`, such as `/exports/a.csv`, while IOStreams v3.0 joins it as a key within the bucket, `exports/a.csv`, as
|
|
224
|
+
# a name joined to the local directory `/` is within it. So it is deprecated, and reports a deprecation warning,
|
|
225
|
+
# see `IOStreams::Deprecation`.
|
|
226
|
+
def join(*elements)
|
|
227
|
+
joined = super
|
|
228
|
+
if path.empty? && joined.path.start_with?("/")
|
|
229
|
+
Deprecation.warn("joining a name that starts with / to an S3 bucket is deprecated, since v3.0 joins it as " \
|
|
230
|
+
"a key within the bucket, such as exports/a.csv for /exports/a.csv: remove the leading /, " \
|
|
231
|
+
"or name a key that starts with / in the url, such as s3://bucket//exports/a.csv")
|
|
232
|
+
end
|
|
233
|
+
joined
|
|
189
234
|
end
|
|
190
235
|
|
|
191
236
|
# Does not support relative file names since there is no concept of current working directory
|
|
@@ -227,9 +272,10 @@ module IOStreams
|
|
|
227
272
|
target = IOStreams.new(target_path)
|
|
228
273
|
return super(target, convert: convert, **args) unless target.is_a?(self.class)
|
|
229
274
|
|
|
275
|
+
warn_unconverted_copy_options(args[:mode], args.except(:mode))
|
|
230
276
|
authorize!
|
|
231
277
|
target.authorize!
|
|
232
|
-
source_name = ::
|
|
278
|
+
source_name = Utils::PathName.join(bucket_name, path)
|
|
233
279
|
client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
|
|
234
280
|
target
|
|
235
281
|
end
|
|
@@ -243,9 +289,10 @@ module IOStreams
|
|
|
243
289
|
return super(source, convert: convert, **args)
|
|
244
290
|
end
|
|
245
291
|
|
|
292
|
+
warn_unconverted_copy_options(args[:mode], args.except(:mode))
|
|
246
293
|
authorize!
|
|
247
294
|
source.authorize!
|
|
248
|
-
source_name = ::
|
|
295
|
+
source_name = Utils::PathName.join(source.bucket_name, source.path)
|
|
249
296
|
client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
|
|
250
297
|
end
|
|
251
298
|
|
|
@@ -339,7 +386,7 @@ module IOStreams
|
|
|
339
386
|
each_object(prefix) do |name, object|
|
|
340
387
|
next if !directories && object.key.end_with?("/")
|
|
341
388
|
|
|
342
|
-
file_name = ::
|
|
389
|
+
file_name = Utils::PathName.join("s3://", name, object.key)
|
|
343
390
|
next unless matcher.match?(file_name)
|
|
344
391
|
|
|
345
392
|
child = child_path(name, object.key)
|
|
@@ -362,6 +409,131 @@ module IOStreams
|
|
|
362
409
|
|
|
363
410
|
private
|
|
364
411
|
|
|
412
|
+
# Reports a deprecation warning for each part of the url that this path cannot use, which IOStreams 3.0 raises
|
|
413
|
+
# ArgumentError for when the path is created, so that a mistake in a configured url is reported wherever the
|
|
414
|
+
# code runs: a url without a bucket, a fragment, which an unencoded `#` starts, and which is not part of the key, a
|
|
415
|
+
# port, and a user name or password, which are ignored.
|
|
416
|
+
def warn_unusable_url(uri)
|
|
417
|
+
warn_url("it names no bucket, name one, such as 's3://bucket/#{path}'") if bucket_name.to_s.empty?
|
|
418
|
+
if uri.fragment
|
|
419
|
+
warn_url("it has a fragment, which a '#' starts, and which is not part of the key, percent-encode a '#' in a " \
|
|
420
|
+
"key as '%23'")
|
|
421
|
+
end
|
|
422
|
+
if uri.port
|
|
423
|
+
warn_url("its port is ignored, supply the endpoint of another service that implements the S3 API with " \
|
|
424
|
+
"client: {endpoint: \"https://host:port\"}")
|
|
425
|
+
end
|
|
426
|
+
return unless uri.user || uri.password
|
|
427
|
+
|
|
428
|
+
warn_url("its user name and password are ignored, supply the credentials with access_key_id: and " \
|
|
429
|
+
"secret_access_key:")
|
|
430
|
+
end
|
|
431
|
+
|
|
432
|
+
# Reports that the S3 url of this path is deprecated, since IOStreams 3.0 raises ArgumentError for it, for the
|
|
433
|
+
# supplied reason.
|
|
434
|
+
def warn_url(reason)
|
|
435
|
+
Deprecation.warn("the S3 url #{warning_url} is deprecated, since #{reason}")
|
|
436
|
+
end
|
|
437
|
+
|
|
438
|
+
# Returns [String] the url of this path as a deprecation warning names it, quoted: its bucket and its key, without
|
|
439
|
+
# the parts of the url that the path cannot use, such as a user name, and with an empty bucket kept, unlike #to_s,
|
|
440
|
+
# such as `"s3:///a.csv"`.
|
|
441
|
+
def warning_url
|
|
442
|
+
Utils.display_text("s3://#{bucket_name}/#{path}").inspect
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# Reports a deprecation warning for the region and credentials supplied with an `Aws::S3::Client`, which ignores
|
|
446
|
+
# them, since it has its own, and which IOStreams 3.0 raises ArgumentError for when the path is created.
|
|
447
|
+
def warn_client_arguments(client, **arguments)
|
|
448
|
+
return if client.nil? || client.is_a?(Hash)
|
|
449
|
+
|
|
450
|
+
names = arguments.compact.keys
|
|
451
|
+
return if names.empty?
|
|
452
|
+
|
|
453
|
+
words = Deprecation::Words.new(names)
|
|
454
|
+
Deprecation.warn("#{words.keywords} #{words.are} deprecated with an Aws::S3::Client as client:, which ignores " \
|
|
455
|
+
"#{words.them}, since it has its own, supply #{words.them} to Aws::S3::Client.new instead, or " \
|
|
456
|
+
"supply client: as a Hash of its options")
|
|
457
|
+
end
|
|
458
|
+
|
|
459
|
+
# Reports a deprecation warning for the options of this path that IOStreams 3.0 raises ArgumentError for when the
|
|
460
|
+
# path is created, see #unusable_option_names: one that the path sets itself, such as `key:`, which it overrides,
|
|
461
|
+
# and one that no S3 request takes, such as a misspelled one, which the AWS SDK raises for once the path is used.
|
|
462
|
+
def warn_unusable_options(names)
|
|
463
|
+
unusable_option_names(names).each_pair do |set_by_path, unusable|
|
|
464
|
+
words = Deprecation::Words.new(unusable)
|
|
465
|
+
warn_unusable("the S3 #{words.one? ? 'option' : 'options'} #{words.keywords}", words, set_by_path)
|
|
466
|
+
end
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
# Reports a deprecation warning for the names in the query of the url of this path that IOStreams 3.0 raises
|
|
470
|
+
# ArgumentError for when the path is created, as for an option, see #warn_unusable_options. A name without a
|
|
471
|
+
# value is more likely part of a key, such as the `?` of `s3://bucket/data/file?.csv`, so the warning then also
|
|
472
|
+
# says how to write a `?` in a key.
|
|
473
|
+
def warn_unusable_query(query)
|
|
474
|
+
unusable_option_names(query.keys.map(&:to_sym)).each_pair do |set_by_path, unusable|
|
|
475
|
+
names = unusable.map(&:to_s)
|
|
476
|
+
words = Deprecation::Words.new(names)
|
|
477
|
+
encode = ", and percent-encode a '?' in a key as '%3F'" if names.any? { |name| query[name].to_s.empty? }
|
|
478
|
+
warn_unusable("#{words.keywords} in the query of the S3 url #{warning_url}", words, set_by_path, encode)
|
|
479
|
+
end
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
# Returns [Hash<true|false, Array>] the supplied names of options that IOStreams 3.0 raises ArgumentError for: by
|
|
483
|
+
# whether the path sets them itself (true), see PATH_PARAMETERS, or no S3 request of a path takes them (false),
|
|
484
|
+
# see .known_option_names, without either when there are none.
|
|
485
|
+
def unusable_option_names(names)
|
|
486
|
+
set_by_path = names & PATH_PARAMETERS
|
|
487
|
+
{true => set_by_path, false => names - set_by_path - self.class.known_option_names}.reject { |_, list| list.empty? }
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
# Reports that the options, which the subject names, and the words are about, see `IOStreams::Deprecation::Words`,
|
|
491
|
+
# are deprecated, since the path sets them itself, or since no S3 request takes them, followed by the supplied
|
|
492
|
+
# advice.
|
|
493
|
+
def warn_unusable(subject, words, set_by_path, advice = nil)
|
|
494
|
+
them = words.them
|
|
495
|
+
reason =
|
|
496
|
+
if set_by_path
|
|
497
|
+
"the path sets #{them} itself, and overrides #{them}, so remove #{them}"
|
|
498
|
+
else
|
|
499
|
+
"no S3 request takes #{them}, so correct or remove #{them}"
|
|
500
|
+
end
|
|
501
|
+
Deprecation.warn("#{subject} #{words.are} deprecated, since #{reason}#{advice}")
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
# Returns [Array(Array<Symbol>, String)|nil] the methods of the IO that v3.0 reads the object from, in place of the
|
|
505
|
+
# temp File that v2.2 downloads it into, and its name, see `IOStreams::Stream#v3_source`: v3.0 reads the object as
|
|
506
|
+
# the application reads it, with an IO that only reads, unless the `range` or `part_number` option chooses the
|
|
507
|
+
# bytes to read, which it still downloads into a temp file.
|
|
508
|
+
def v3_source
|
|
509
|
+
return if options.key?(:range) || options.key?(:part_number)
|
|
510
|
+
|
|
511
|
+
[IOStreams::Reader::V3_READ_METHODS, "the IO of an S3 path"]
|
|
512
|
+
end
|
|
513
|
+
|
|
514
|
+
# The methods of the IO that v3.0 uploads the object with, see #v3_target: those of an IO that write, and #flush.
|
|
515
|
+
V3_UPLOAD_METHODS = [*IOStreams::Writer::V3_WRITE_METHODS, :flush].freeze
|
|
516
|
+
private_constant :V3_UPLOAD_METHODS
|
|
517
|
+
|
|
518
|
+
# The options that describe the whole object, with which v3.0 still writes the object into a temp file first, as
|
|
519
|
+
# does a checksum of its data, such as `checksum_crc32`, see #v3_target.
|
|
520
|
+
V3_WHOLE_OBJECT_OPTIONS = %i[content_length content_md5 write_offset_bytes].freeze
|
|
521
|
+
private_constant :V3_WHOLE_OBJECT_OPTIONS
|
|
522
|
+
|
|
523
|
+
# Returns [Array(Array<Symbol>, String)|nil] the methods of the IO that v3.0 uploads the object with, in place of the
|
|
524
|
+
# temp File that v2.2 writes it into, and its name, see `IOStreams::Stream#v3_target`: v3.0 uploads the object as
|
|
525
|
+
# the application writes it, with an IO that only writes, unless an option describes the whole object, which it
|
|
526
|
+
# still writes into a temp file first. v3.0 writes the object with only the options that `put_object` takes, so
|
|
527
|
+
# only those decide, rather than one that only another request takes, such as `checksum_mode` when reading.
|
|
528
|
+
def v3_target
|
|
529
|
+
put_object = ::Aws::S3::Client.api.operation(:put_object).input.shape
|
|
530
|
+
whole_object = options.each_key.any? do |name|
|
|
531
|
+
put_object.member?(name) &&
|
|
532
|
+
(V3_WHOLE_OBJECT_OPTIONS.include?(name) || (name.start_with?("checksum_") && name != :checksum_algorithm))
|
|
533
|
+
end
|
|
534
|
+
[V3_UPLOAD_METHODS, "the IO of an S3 path"] unless whole_object
|
|
535
|
+
end
|
|
536
|
+
|
|
365
537
|
# Yields the bucket name and each object in the bucket whose key starts with the supplied prefix.
|
|
366
538
|
def each_object(prefix)
|
|
367
539
|
token = nil
|
|
@@ -391,9 +563,8 @@ module IOStreams
|
|
|
391
563
|
#
|
|
392
564
|
# The child uses this path's client, so that it has the same credentials and region.
|
|
393
565
|
def child_path(bucket_name, key)
|
|
394
|
-
child
|
|
395
|
-
child.
|
|
396
|
-
child
|
|
566
|
+
child = self.class.new("s3://#{bucket_name}", client: client)
|
|
567
|
+
child.replace_path(key.dup.freeze)
|
|
397
568
|
end
|
|
398
569
|
end
|
|
399
570
|
end
|
|
@@ -36,6 +36,20 @@ module IOStreams
|
|
|
36
36
|
|
|
37
37
|
attr_reader :hostname, :username, :ssh_options, :url, :port
|
|
38
38
|
|
|
39
|
+
# The password, and the private key supplied as the ssh option `IdentityKey`, see IOStreams::Path.redact_options.
|
|
40
|
+
def self.sensitive_option_names = %i[password identity_key]
|
|
41
|
+
|
|
42
|
+
# Returns [String|nil] the deprecation warning for an SFTP url without a path, such as `sftp://host`, which
|
|
43
|
+
# IOStreams v3.0 reads as `sftp://host/`, the root directory, see `IOStreams::Path.name_deprecation`, while this
|
|
44
|
+
# version lists the login directory for it, see #each_child.
|
|
45
|
+
def self.name_deprecation(url)
|
|
46
|
+
return unless url_without_path?(url)
|
|
47
|
+
|
|
48
|
+
"an SFTP url without a path, such as sftp://host, is deprecated, since v3.0 reads it as sftp://host/, the " \
|
|
49
|
+
"root directory, which changes its #to_s and #path, and what #each_child lists: supply the url with the /, " \
|
|
50
|
+
"which v2.2 reads the same way"
|
|
51
|
+
end
|
|
52
|
+
|
|
39
53
|
# Stream to a remote file over sftp.
|
|
40
54
|
#
|
|
41
55
|
# url: [String]
|
|
@@ -44,7 +58,8 @@ module IOStreams
|
|
|
44
58
|
# SECURITY WARNING:
|
|
45
59
|
# A username and password supplied in the url remain part of it, so `#to_s` and `#url`
|
|
46
60
|
# return them, as does any log or error message that includes the path.
|
|
47
|
-
# Supply them with the `username:` and `password:` arguments instead
|
|
61
|
+
# Supply them with the `username:` and `password:` arguments instead,
|
|
62
|
+
# and log `#display_name`, which never includes them.
|
|
48
63
|
#
|
|
49
64
|
# username: [String]
|
|
50
65
|
# Name of user to login with.
|
|
@@ -117,6 +132,7 @@ module IOStreams
|
|
|
117
132
|
validate_username!
|
|
118
133
|
|
|
119
134
|
super(uri.path)
|
|
135
|
+
warn_unusable_url(uri)
|
|
120
136
|
end
|
|
121
137
|
|
|
122
138
|
# Does not support relative file names since there is no concept of current working directory
|
|
@@ -128,6 +144,11 @@ module IOStreams
|
|
|
128
144
|
url
|
|
129
145
|
end
|
|
130
146
|
|
|
147
|
+
# Returns [String] the url without the user name, password, query or fragment, which can hold credentials.
|
|
148
|
+
def display_name
|
|
149
|
+
url.sub(%r{\A([^:/]+://)[^/?#]*@}, "\\1").sub(/[?#].*\z/m, "")
|
|
150
|
+
end
|
|
151
|
+
|
|
131
152
|
# Note that mkdir is delayed and only executed when the file write is performed.
|
|
132
153
|
def mkdir
|
|
133
154
|
@mkdir = true
|
|
@@ -194,6 +215,29 @@ module IOStreams
|
|
|
194
215
|
"sftp://#{hostname.to_s.downcase}:#{port}#{normalize_path(path)}".chomp("/")
|
|
195
216
|
end
|
|
196
217
|
|
|
218
|
+
# Reports a deprecation warning for each part of the url that this path cannot use, which IOStreams 3.0 raises
|
|
219
|
+
# ArgumentError for when the path is created, so that a mistake in a configured url is reported wherever the
|
|
220
|
+
# code runs: a url without a host, a query, whose ssh options are ignored, and a fragment, which an unencoded `#`
|
|
221
|
+
# starts, and which is not part of the name. Each warning names the url without its user name, password, query or
|
|
222
|
+
# fragment, see #display_name.
|
|
223
|
+
def warn_unusable_url(uri)
|
|
224
|
+
warn_url("it names no host, name one, such as 'sftp://example.org#{path}'") if uri.hostname.to_s.empty?
|
|
225
|
+
if uri.query
|
|
226
|
+
warn_url("its query is ignored, supply ssh options with ssh_options:, and percent-encode a '?' in a name as " \
|
|
227
|
+
"'%3F'")
|
|
228
|
+
end
|
|
229
|
+
return unless uri.fragment
|
|
230
|
+
|
|
231
|
+
warn_url("it has a fragment, which a '#' starts, and which is not part of the name, percent-encode a '#' in a " \
|
|
232
|
+
"name as '%23'")
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
# Reports that the url of this path is deprecated, since IOStreams 3.0 raises ArgumentError for it, for the
|
|
236
|
+
# supplied reason.
|
|
237
|
+
def warn_url(reason)
|
|
238
|
+
Deprecation.warn("the SFTP url #{display_name.inspect} is deprecated, since #{reason}")
|
|
239
|
+
end
|
|
240
|
+
|
|
197
241
|
# Usernames are passed to the `sftp` executable, so reject values that it could treat as options.
|
|
198
242
|
def validate_username!
|
|
199
243
|
return if username.nil?
|
|
@@ -209,8 +253,8 @@ module IOStreams
|
|
|
209
253
|
def child_path(name)
|
|
210
254
|
server = port == 22 ? "sftp://#{hostname}" : "sftp://#{hostname}:#{port}"
|
|
211
255
|
child = self.class.new(server, username: username, password: password, ssh_options: ssh_options)
|
|
212
|
-
child.
|
|
213
|
-
child.url
|
|
256
|
+
child.replace_path((path.empty? ? "/#{name}" : Utils::PathName.join(path, name)).freeze)
|
|
257
|
+
child.url = "#{server}#{child.path}"
|
|
214
258
|
child
|
|
215
259
|
end
|
|
216
260
|
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
module IOStreams
|
|
2
|
+
# The path classes of the schemes of urls, such as `IOStreams::Paths::S3` for `s3://`, see `IOStreams.register_scheme`.
|
|
3
|
+
module Paths
|
|
4
|
+
# Returns [IOStreams::Path] the path of the supplied name or url, created with the supplied arguments by the path
|
|
5
|
+
# class of its scheme, see .path_class.
|
|
6
|
+
#
|
|
7
|
+
# It reports no deprecation warning for the name, which `IOStreams.path` reports for a name that the application
|
|
8
|
+
# supplies, see `IOStreams::Path.name_deprecation`, so that IOStreams can create the path of a name that it derives
|
|
9
|
+
# from one, such as the directory that a pattern is matched within, see `IOStreams::Paths::Matcher`.
|
|
10
|
+
def self.build(name, **args)
|
|
11
|
+
klass = path_class(name)
|
|
12
|
+
args.empty? ? klass.new(name) : klass.new(name, **args)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Returns [Class] the path class of the scheme of the supplied name or url, such as `IOStreams::Paths::SFTP`, or the
|
|
16
|
+
# one registered for a name without a scheme, such as `IOStreams::Paths::File`, see `IOStreams.scheme`.
|
|
17
|
+
#
|
|
18
|
+
# Raises ArgumentError when the scheme is not registered, and `URI::Error` when the url is not valid.
|
|
19
|
+
def self.path_class(name)
|
|
20
|
+
IOStreams.scheme(name.include?("://") ? Utils::URI.new(name).scheme : nil)
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -7,6 +7,24 @@ module IOStreams
|
|
|
7
7
|
%i[passphrase ignore_mdc_error verify_first]
|
|
8
8
|
end
|
|
9
9
|
|
|
10
|
+
def self.sensitive_option_names
|
|
11
|
+
%i[passphrase]
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
# The methods that the stream that v3.0 supplies to a reader block for PGP answers, when gpg does not read a regular
|
|
15
|
+
# local file, see .v3_stream_methods: those of an IO that read, and #binmode and #binmode?.
|
|
16
|
+
V3_METHODS = [*V3_READ_METHODS, :binmode, :binmode?].freeze
|
|
17
|
+
private_constant :V3_METHODS
|
|
18
|
+
|
|
19
|
+
# Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a reader block, in place of the IO
|
|
20
|
+
# of gpg's output that this reader yields, answers, see `IOStreams::Reader.v3_stream_methods`: nil with
|
|
21
|
+
# `verify_first: true`, since both versions then yield the File that gpg decrypts the data into, or when it reads
|
|
22
|
+
# a regular local file, which gpg reads itself in v3.0, so that the block receives gpg's output in both versions,
|
|
23
|
+
# otherwise those of the stream that v3.0 supplies, which feeds gpg the data that it reads.
|
|
24
|
+
def self.v3_stream_methods(options)
|
|
25
|
+
V3_METHODS unless options[:verify_first] || yield
|
|
26
|
+
end
|
|
27
|
+
|
|
10
28
|
# Passphrase to use to open the private key to decrypt the received file
|
|
11
29
|
class << self
|
|
12
30
|
attr_writer :default_passphrase
|
|
@@ -11,6 +11,22 @@ module IOStreams
|
|
|
11
11
|
compress compress_level]
|
|
12
12
|
end
|
|
13
13
|
|
|
14
|
+
def self.sensitive_option_names
|
|
15
|
+
%i[signer_passphrase]
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# The methods that the stream that v3.0 supplies to a writer block for PGP answers: those of an IO that write,
|
|
19
|
+
# #flush, #binmode and #binmode?, see .v3_stream_methods.
|
|
20
|
+
V3_METHODS = [*V3_WRITE_METHODS, :flush, :binmode, :binmode?].freeze
|
|
21
|
+
private_constant :V3_METHODS
|
|
22
|
+
|
|
23
|
+
# Returns [Array<Symbol>] the methods that the stream that v3.0 supplies to a writer block, in place of the IO of
|
|
24
|
+
# gpg's input that this writer yields, answers, which writes to gpg, whatever it writes to, see
|
|
25
|
+
# `IOStreams::Writer.v3_stream_methods`.
|
|
26
|
+
def self.v3_stream_methods(_options)
|
|
27
|
+
V3_METHODS
|
|
28
|
+
end
|
|
29
|
+
|
|
14
30
|
class << self
|
|
15
31
|
# Sign all encrypted files with this users key.
|
|
16
32
|
# Default: Do not sign encrypted files.
|
data/lib/io_streams/pgp.rb
CHANGED
|
@@ -262,8 +262,9 @@ module IOStreams
|
|
|
262
262
|
# Returns [String] containing all the public keys for the supplied email address or key id.
|
|
263
263
|
#
|
|
264
264
|
# Raises ArgumentError when neither :email nor :key_id is supplied.
|
|
265
|
-
# Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added
|
|
266
|
-
#
|
|
265
|
+
# Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added, which is
|
|
266
|
+
# deprecated, since v3.0 raises ArgumentError for it instead: it reports a deprecation warning, see
|
|
267
|
+
# `IOStreams::Deprecation`.
|
|
267
268
|
#
|
|
268
269
|
# email: [String] Email address for requested key.
|
|
269
270
|
#
|
|
@@ -276,6 +277,9 @@ module IOStreams
|
|
|
276
277
|
if email.nil? && key_id.nil?
|
|
277
278
|
raise(ArgumentError, "Either :email, or :key_id must be supplied") if email_omitted
|
|
278
279
|
|
|
280
|
+
Deprecation.warn("IOStreams::Pgp.export with email: nil and without a key_id: is deprecated, since v3.0 " \
|
|
281
|
+
"raises ArgumentError for it rather than IOStreams::Pgp::Failure: supply the email: or " \
|
|
282
|
+
"key_id: of the key")
|
|
279
283
|
raise(Pgp::Failure, "GPG Failed reading key: no email address or key id was supplied")
|
|
280
284
|
end
|
|
281
285
|
|
|
@@ -490,6 +494,10 @@ module IOStreams
|
|
|
490
494
|
#
|
|
491
495
|
# After importing keys, they are not trusted and the relevant trust level must be set.
|
|
492
496
|
#
|
|
497
|
+
# Without an :email or :key_id, it raises Pgp::Failure, since gpg rejects the empty user id that it is asked for,
|
|
498
|
+
# which is deprecated, since v3.0 raises ArgumentError instead: it reports a deprecation warning, see
|
|
499
|
+
# `IOStreams::Deprecation`.
|
|
500
|
+
#
|
|
493
501
|
# key_id: [String]
|
|
494
502
|
# The fingerprint of the key, as hexadecimal digits only.
|
|
495
503
|
# Raises ArgumentError when it contains any other characters.
|
|
@@ -511,6 +519,12 @@ module IOStreams
|
|
|
511
519
|
# key at this level allows that attacker to impersonate other recipients.
|
|
512
520
|
# When the key cannot be fully verified, supply a lower `level`.
|
|
513
521
|
def self.set_trust(email: nil, key_id: nil, level: 5)
|
|
522
|
+
if email.nil? && key_id.nil?
|
|
523
|
+
Deprecation.warn("IOStreams::Pgp.set_trust without an email: or key_id: is deprecated, since v3.0 raises " \
|
|
524
|
+
"ArgumentError for it rather than IOStreams::Pgp::Failure: supply the email: or key_id: of " \
|
|
525
|
+
"the key")
|
|
526
|
+
end
|
|
527
|
+
|
|
514
528
|
# The key_id is written into gpg's ownertrust input, where any other character, such as a newline,
|
|
515
529
|
# could add trust lines for other keys.
|
|
516
530
|
if key_id && !key_id.to_s.match?(/\A\h+\z/)
|
data/lib/io_streams/reader.rb
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
1
|
module IOStreams
|
|
2
2
|
class Reader
|
|
3
|
+
# Answers the names of the options that this reader takes in IOStreams 3.0, and those that it replaces.
|
|
4
|
+
extend Deprecation::OptionNames
|
|
5
|
+
|
|
3
6
|
# Returns [Array<Symbol>] the names of the options this reader accepts,
|
|
4
7
|
# or [nil] when the reader does not declare them.
|
|
5
8
|
#
|
|
@@ -9,6 +12,47 @@ module IOStreams
|
|
|
9
12
|
nil
|
|
10
13
|
end
|
|
11
14
|
|
|
15
|
+
# Returns [Array<Symbol>] the names of the options whose values must not be displayed, such as a passphrase,
|
|
16
|
+
# so that `#inspect` on a path does not display them, see `IOStreams::Extension#redact_options`.
|
|
17
|
+
def self.sensitive_option_names
|
|
18
|
+
[]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Returns [Symbol|nil] the tabular format of the text that this reader turns its data into, such as `:csv` for a
|
|
22
|
+
# spreadsheet, otherwise nil, for a reader whose data is the text, such as gzip, see
|
|
23
|
+
# `IOStreams::Builder#text_format`.
|
|
24
|
+
def self.text_format
|
|
25
|
+
nil
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# The methods of an IO that read, which every stream that v3.0 supplies to a reader block answers, see
|
|
29
|
+
# .v3_stream_methods.
|
|
30
|
+
V3_READ_METHODS = %i[read readpartial gets readline readlines each_line each eof? eof closed?].freeze
|
|
31
|
+
|
|
32
|
+
# Returns [Array<Symbol>|nil] the methods that the stream that v3.0 supplies to a reader block, in place of the one
|
|
33
|
+
# that this reader yields, answers, when it does not answer all of the methods of that one, such as when it is the
|
|
34
|
+
# reader of a gem, otherwise nil, for a reader that yields the same kind of stream in both versions, see
|
|
35
|
+
# `IOStreams::Builder#v3_stream`.
|
|
36
|
+
#
|
|
37
|
+
# Parameters:
|
|
38
|
+
# options: [Hash]
|
|
39
|
+
# The options that this reader is opened with, such as `verify_first: true` for PGP, see
|
|
40
|
+
# `IOStreams::Pgp::Reader.v3_stream_methods`.
|
|
41
|
+
#
|
|
42
|
+
# Yields, for a reader whose stream depends on it, to ask whether this reader reads a regular local file in v3.0,
|
|
43
|
+
# rather than another stream, an IO that is not a file, or the object of an S3 or HTTP path, which v3.0 reads without
|
|
44
|
+
# a temp file. The block answers true or false.
|
|
45
|
+
def self.v3_stream_methods(_options)
|
|
46
|
+
nil
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
# Returns [Hash<Symbol:String>|nil] what the application uses in place of each method of the stream that this reader
|
|
50
|
+
# yields that v3.0 removes, when it is not the methods that the stream of v3.0 answers, see .v3_stream_methods, such as
|
|
51
|
+
# a setting of the path, otherwise nil, see `IOStreams::Builder#v3_stream`.
|
|
52
|
+
def self.v3_stream_advice
|
|
53
|
+
nil
|
|
54
|
+
end
|
|
55
|
+
|
|
12
56
|
# When a Reader does not support streams, we copy the stream to a local temp file
|
|
13
57
|
# and then pass that filename in for this reader.
|
|
14
58
|
def self.stream(input_stream, **args, &block)
|