iostreams 1.11.0 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (92) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +20 -2
  3. data/Rakefile +7 -0
  4. data/lib/io_streams/builder.rb +9 -9
  5. data/lib/io_streams/bzip2/writer.rb +1 -1
  6. data/lib/io_streams/encode/reader.rb +2 -2
  7. data/lib/io_streams/encode/writer.rb +5 -5
  8. data/lib/io_streams/gzip/reader.rb +1 -1
  9. data/lib/io_streams/gzip/writer.rb +1 -1
  10. data/lib/io_streams/io_streams.rb +45 -19
  11. data/lib/io_streams/line/reader.rb +2 -2
  12. data/lib/io_streams/line/writer.rb +1 -1
  13. data/lib/io_streams/path.rb +2 -2
  14. data/lib/io_streams/paths/file.rb +10 -10
  15. data/lib/io_streams/paths/http.rb +80 -7
  16. data/lib/io_streams/paths/matcher.rb +3 -3
  17. data/lib/io_streams/paths/s3.rb +3 -3
  18. data/lib/io_streams/paths/sftp.rb +7 -8
  19. data/lib/io_streams/pgp/reader.rb +23 -10
  20. data/lib/io_streams/pgp/writer.rb +93 -32
  21. data/lib/io_streams/pgp.rb +188 -60
  22. data/lib/io_streams/reader.rb +4 -4
  23. data/lib/io_streams/record/reader.rb +3 -4
  24. data/lib/io_streams/record/writer.rb +3 -4
  25. data/lib/io_streams/row/reader.rb +1 -1
  26. data/lib/io_streams/row/writer.rb +1 -1
  27. data/lib/io_streams/stream.rb +36 -30
  28. data/lib/io_streams/symmetric_encryption/reader.rb +2 -2
  29. data/lib/io_streams/symmetric_encryption/writer.rb +4 -4
  30. data/lib/io_streams/tabular/header.rb +18 -6
  31. data/lib/io_streams/tabular/parser/array.rb +0 -10
  32. data/lib/io_streams/tabular/parser/csv.rb +6 -38
  33. data/lib/io_streams/tabular/parser/fixed.rb +5 -5
  34. data/lib/io_streams/tabular/parser/psv.rb +0 -12
  35. data/lib/io_streams/tabular.rb +5 -10
  36. data/lib/io_streams/utils.rb +3 -2
  37. data/lib/io_streams/version.rb +1 -1
  38. data/lib/io_streams/writer.rb +6 -6
  39. data/lib/io_streams/xlsx/reader.rb +1 -1
  40. data/lib/io_streams/zip/writer.rb +22 -10
  41. data/lib/iostreams.rb +0 -1
  42. metadata +28 -111
  43. data/lib/io_streams/deprecated.rb +0 -216
  44. data/lib/io_streams/tabular/utility/csv_row.rb +0 -105
  45. data/test/builder_test.rb +0 -311
  46. data/test/bzip2_reader_test.rb +0 -27
  47. data/test/bzip2_writer_test.rb +0 -56
  48. data/test/deprecated_test.rb +0 -121
  49. data/test/encode_reader_test.rb +0 -51
  50. data/test/encode_writer_test.rb +0 -90
  51. data/test/files/embedded_lines_test.csv +0 -7
  52. data/test/files/multiple_files.zip +0 -0
  53. data/test/files/spreadsheet.xlsx +0 -0
  54. data/test/files/test.csv +0 -4
  55. data/test/files/test.json +0 -3
  56. data/test/files/test.psv +0 -4
  57. data/test/files/text file.txt +0 -3
  58. data/test/files/text.txt +0 -3
  59. data/test/files/text.txt.bz2 +0 -0
  60. data/test/files/text.txt.gz +0 -0
  61. data/test/files/text.txt.gz.zip +0 -0
  62. data/test/files/text.zip +0 -0
  63. data/test/files/text.zip.gz +0 -0
  64. data/test/files/unclosed_quote_large_test.csv +0 -1658
  65. data/test/files/unclosed_quote_test.csv +0 -4
  66. data/test/files/unclosed_quote_test2.csv +0 -3
  67. data/test/gzip_reader_test.rb +0 -27
  68. data/test/gzip_writer_test.rb +0 -52
  69. data/test/io_streams_test.rb +0 -132
  70. data/test/line_reader_test.rb +0 -325
  71. data/test/line_writer_test.rb +0 -59
  72. data/test/minimal_file_reader.rb +0 -25
  73. data/test/path_test.rb +0 -55
  74. data/test/paths/file_test.rb +0 -213
  75. data/test/paths/http_test.rb +0 -34
  76. data/test/paths/matcher_test.rb +0 -120
  77. data/test/paths/s3_test.rb +0 -220
  78. data/test/paths/sftp_test.rb +0 -106
  79. data/test/pgp_reader_test.rb +0 -46
  80. data/test/pgp_test.rb +0 -267
  81. data/test/pgp_writer_test.rb +0 -130
  82. data/test/record_reader_test.rb +0 -60
  83. data/test/record_writer_test.rb +0 -82
  84. data/test/row_reader_test.rb +0 -35
  85. data/test/row_writer_test.rb +0 -56
  86. data/test/stream_test.rb +0 -577
  87. data/test/tabular_test.rb +0 -338
  88. data/test/test_helper.rb +0 -40
  89. data/test/utils_test.rb +0 -20
  90. data/test/xlsx_reader_test.rb +0 -37
  91. data/test/zip_reader_test.rb +0 -53
  92. data/test/zip_writer_test.rb +0 -48
@@ -1,4 +1,5 @@
1
1
  require "open3"
2
+ require "shellwords"
2
3
  module IOStreams
3
4
  # Read/Write PGP/GPG file or stream.
4
5
  #
@@ -25,6 +26,18 @@ module IOStreams
25
26
 
26
27
  @executable = "gpg"
27
28
 
29
+ # Returns [Array<String>] the argv used to invoke gpg, with the supplied
30
+ # arguments appended.
31
+ #
32
+ # All gpg invocations are run without a shell (the multi-argument form of
33
+ # `Open3`) so that values such as email addresses, key ids, passphrases and
34
+ # file names cannot be interpreted as shell commands. The configured
35
+ # `executable` is split with `Shellwords` so that it may still contain
36
+ # additional fixed arguments (for example "gpg --homedir /path").
37
+ def self.gpg_command(*args)
38
+ Shellwords.split(executable) + args.map(&:to_s)
39
+ end
40
+
28
41
  # Generate a new ultimate trusted local public and private key.
29
42
  #
30
43
  # Returns [String] the key id for the generated key.
@@ -44,6 +57,20 @@ module IOStreams
44
57
  # Highly Recommended.
45
58
  # To generate a good passphrase:
46
59
  # `SecureRandom.urlsafe_base64(128)`
60
+ # Pass `nil` to generate an unprotected (passphrase-less) key.
61
+ #
62
+ # key_curve / subkey_curve [String]
63
+ # Optional Elliptic Curve to use for the (sub)key, e.g. "ed25519".
64
+ # When supplied the corresponding key/subkey length is ignored.
65
+ # Requires GnuPG 2.1 or later.
66
+ #
67
+ # key_usage / subkey_usage [String]
68
+ # Optional comma separated list of (sub)key capabilities, e.g. "sign".
69
+ # Requires GnuPG 2.1 or later.
70
+ #
71
+ # creation_date [String]
72
+ # Optional creation date for the key, e.g. "20240101T000000".
73
+ # Requires GnuPG 2.1 or later.
47
74
  #
48
75
  # See `man gpg` for the remaining options
49
76
  def self.generate_key(name:,
@@ -54,23 +81,70 @@ module IOStreams
54
81
  key_length: 4096,
55
82
  subkey_type: "RSA",
56
83
  subkey_length: key_length,
84
+ key_curve: nil,
85
+ key_usage: nil,
86
+ subkey_curve: nil,
87
+ subkey_usage: nil,
88
+ creation_date: nil,
57
89
  expire_date: nil)
58
90
  version_check
59
- params = ""
91
+
92
+ # Reject newlines so that a value cannot inject additional directives into
93
+ # the gpg batch key-generation parameter file.
94
+ reject_newlines!(name: name, email: email, comment: comment, passphrase: passphrase,
95
+ key_type: key_type, subkey_type: subkey_type, expire_date: expire_date,
96
+ key_curve: key_curve, key_usage: key_usage,
97
+ subkey_curve: subkey_curve, subkey_usage: subkey_usage,
98
+ creation_date: creation_date)
99
+
100
+ # `%no-protection`, and the Elliptic Curve / usage / creation-date directives
101
+ # were all introduced in GnuPG 2.1. Keep older versions working by only
102
+ # emitting them when a 2.1+ binary is detected. `--batch --gen-key` accepts
103
+ # all of these on 2.1+, so there is no need for the newer `--full-gen-key`.
104
+ modern = pgp_version.to_f >= 2.1
105
+
106
+ unless modern
107
+ new_options = {
108
+ key_curve: key_curve,
109
+ key_usage: key_usage,
110
+ subkey_curve: subkey_curve,
111
+ subkey_usage: subkey_usage,
112
+ creation_date: creation_date
113
+ }.compact
114
+ unless new_options.empty?
115
+ raise(ArgumentError,
116
+ "IOStreams::Pgp.generate_key: #{new_options.keys.join(', ')} require GnuPG 2.1 or later " \
117
+ "(detected #{pgp_version})")
118
+ end
119
+ end
120
+
121
+ params = +""
122
+ # `%no-protection` is a control statement and must precede the key parameters.
123
+ # GnuPG 2.1+ requires this explicit opt-out to create an unprotected key;
124
+ # older versions create one simply by omitting the Passphrase directive.
125
+ params << "%no-protection\n" if !passphrase && modern
60
126
  params << "Key-Type: #{key_type}\n" if key_type
61
- params << "Key-Length: #{key_length}\n" if key_length
127
+ # Key-Length and Key-Curve are mutually exclusive: curves imply their own length.
128
+ params << "Key-Length: #{key_length}\n" if key_length && !key_curve
129
+ params << "Key-Curve: #{key_curve}\n" if key_curve
130
+ params << "Key-Usage: #{key_usage}\n" if key_usage
62
131
  params << "Subkey-Type: #{subkey_type}\n" if subkey_type
63
- params << "Subkey-Length: #{subkey_length}\n" if subkey_length
132
+ params << "Subkey-Length: #{subkey_length}\n" if subkey_length && !subkey_curve
133
+ params << "Subkey-Curve: #{subkey_curve}\n" if subkey_curve
134
+ params << "Subkey-Usage: #{subkey_usage}\n" if subkey_usage
64
135
  params << "Name-Real: #{name}\n" if name
65
136
  params << "Name-Comment: #{comment}\n" if comment
66
137
  params << "Name-Email: #{email}\n" if email
67
138
  params << "Expire-Date: #{expire_date}\n" if expire_date
139
+ params << "Creation-Date: #{creation_date}\n" if creation_date
68
140
  params << "Passphrase: #{passphrase}\n" if passphrase
69
141
  params << "%commit"
70
- command = "#{executable} --batch --gen-key --no-tty"
71
142
 
72
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: params)
73
- logger&.debug { "IOStreams::Pgp.generate_key: #{command}\n#{params}\n#{err}#{out}" }
143
+ command = gpg_command("--batch", "--gen-key", "--no-tty")
144
+
145
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: params)
146
+ # Do not log `params`, it contains the passphrase.
147
+ IOStreams.logger&.debug { "IOStreams::Pgp.generate_key: #{command.shelljoin}\n#{err}#{out}" }
74
148
 
75
149
  raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
76
150
 
@@ -127,14 +201,17 @@ module IOStreams
127
201
  # date: [String]
128
202
  # name: [String]
129
203
  # email: [String]
204
+ # private: [true|false]
205
+ # trust: [String]
130
206
  # Returns [] if no keys were found.
131
207
  def self.list_keys(email: nil, key_id: nil, private: false)
132
208
  version_check
133
- cmd = private ? "--list-secret-keys" : "--list-keys"
134
- command = "#{executable} #{cmd} #{email || key_id}"
209
+ args = [private ? "--list-secret-keys" : "--list-keys"]
210
+ args << (email || key_id).to_s if email || key_id
211
+ command = gpg_command(*args)
135
212
 
136
- out, err, status = Open3.capture3(command, binmode: true)
137
- logger&.debug { "IOStreams::Pgp.list_keys: #{command}\n#{err}#{out}" }
213
+ out, err, status = Open3.capture3(*command, binmode: true)
214
+ IOStreams.logger&.debug { "IOStreams::Pgp.list_keys: #{command.shelljoin}\n#{err}#{out}" }
138
215
  if status.success? && out.length.positive?
139
216
  parse_list_output(out)
140
217
  else
@@ -156,12 +233,14 @@ module IOStreams
156
233
  # date: [String]
157
234
  # name: [String]
158
235
  # email: [String]
236
+ # private: [true|false]
237
+ # trust: [String]
159
238
  def self.key_info(key:)
160
239
  version_check
161
- command = "#{executable} --batch --no-tty"
240
+ command = gpg_command("--batch", "--no-tty")
162
241
 
163
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
164
- logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
242
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
243
+ IOStreams.logger&.debug { "IOStreams::Pgp.key_info: #{command.shelljoin}\n#{err}#{out}" }
165
244
 
166
245
  # Try parsing even if we get an error - some versions of GPG return non-zero status but still output key info
167
246
  unless (status.success? || err.include?("key ID") || out.include?("pub")) && out.length.positive?
@@ -186,16 +265,18 @@ module IOStreams
186
265
  def self.export(email:, ascii: true, private: false, passphrase: nil)
187
266
  version_check
188
267
 
189
- command = "#{executable} "
190
- command << "--pinentry-mode loopback " if pgp_version.to_f >= 2.1
191
- command << "--no-symkey-cache " if pgp_version.to_f >= 2.4
192
- command << "--armor " if ascii
193
- command << "--no-tty --batch --passphrase"
194
- command << (passphrase ? " #{passphrase} " : "-fd 0 ")
195
- command << (private ? "--export-secret-keys #{email}" : "--export #{email}")
268
+ args = []
269
+ args += ["--pinentry-mode", "loopback"] if pgp_version.to_f >= 2.1
270
+ args << "--no-symkey-cache" if pgp_version.to_f >= 2.4
271
+ args << "--armor" if ascii
272
+ args += ["--no-tty", "--batch"]
273
+ args += passphrase ? ["--passphrase", passphrase] : ["--passphrase-fd", "0"]
274
+ args += private ? ["--export-secret-keys", email.to_s] : ["--export", email.to_s]
275
+ command = gpg_command(*args)
196
276
 
197
- out, err, status = Open3.capture3(command, binmode: true)
198
- logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
277
+ out, err, status = Open3.capture3(*command, binmode: true)
278
+ # Do not log the command, it may contain the passphrase.
279
+ IOStreams.logger&.debug { "IOStreams::Pgp.export: #{email}\n#{err}" }
199
280
 
200
281
  raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
201
282
 
@@ -219,10 +300,10 @@ module IOStreams
219
300
  # * Invalidated keys must be removed manually.
220
301
  def self.import(key:)
221
302
  version_check
222
- command = "#{executable} --batch --import"
303
+ command = gpg_command("--batch", "--import")
223
304
 
224
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
225
- logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
305
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
306
+ IOStreams.logger&.debug { "IOStreams::Pgp.import: #{command.shelljoin}\n#{err}#{out}" }
226
307
 
227
308
  # Handle both old and new versions of GPG
228
309
  # For older versions, the output is in err, for newer ones it might be in out
@@ -316,11 +397,33 @@ module IOStreams
316
397
  raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
317
398
  end
318
399
 
319
- # Returns [String] email for the supplied after importing and trusting the key
400
+ # Imports the supplied key and then marks it as trusted at the supplied trust level.
401
+ #
402
+ # Returns [String] email for the supplied key, or its key id when no email is present.
403
+ #
404
+ # key: [String]
405
+ # The public (or private) key to import and trust.
406
+ #
407
+ # trust_level: [Integer]
408
+ # The owner-trust level to assign to the imported key, the same levels used by `set_trust`:
409
+ # 1 : Undefined (no opinion)
410
+ # 2 : Never (do not trust)
411
+ # 3 : Marginal
412
+ # 4 : Full
413
+ # 5 : Ultimate
414
+ # Default: 5 : Ultimate
415
+ #
416
+ # SECURITY WARNING:
417
+ # Only import and trust keys received from a verified, trusted source.
418
+ # The default trust level is `5` (Ultimate), which tells GPG to treat the imported key
419
+ # as if it were one of your own keys. An ultimately trusted key is implicitly valid and
420
+ # can in turn confer validity on other keys it has signed. Importing an attacker supplied
421
+ # key at this level allows that attacker to impersonate other recipients.
422
+ # When the key cannot be fully verified, supply a lower `trust_level`.
320
423
  #
321
424
  # Notes:
322
425
  # - If the same email address has multiple keys then only the first is currently trusted.
323
- def self.import_and_trust(key:)
426
+ def self.import_and_trust(key:, trust_level: 5)
324
427
  raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
325
428
 
326
429
  key_info = key_info(key: key).last
@@ -330,7 +433,7 @@ module IOStreams
330
433
  raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
331
434
 
332
435
  import(key: key)
333
- set_trust(email: email, key_id: key_id)
436
+ set_trust(email: email, key_id: key_id, level: trust_level)
334
437
  email || key_id
335
438
  end
336
439
 
@@ -340,50 +443,65 @@ module IOStreams
340
443
  # Returns nil if the email was not found
341
444
  #
342
445
  # After importing keys, they are not trusted and the relevant trust level must be set.
446
+ #
447
+ # level: [Integer]
448
+ # The owner-trust level to assign to the key:
449
+ # 1 : Undefined (no opinion)
450
+ # 2 : Never (do not trust)
451
+ # 3 : Marginal
452
+ # 4 : Full
453
+ # 5 : Ultimate
343
454
  # Default: 5 : Ultimate
455
+ #
456
+ # SECURITY WARNING:
457
+ # Only trust keys received from a verified, trusted source.
458
+ # The default trust level is `5` (Ultimate), which tells GPG to treat the key
459
+ # as if it were one of your own keys. An ultimately trusted key is implicitly valid and
460
+ # can in turn confer validity on other keys it has signed. Trusting an attacker supplied
461
+ # key at this level allows that attacker to impersonate other recipients.
462
+ # When the key cannot be fully verified, supply a lower `level`.
344
463
  def self.set_trust(email: nil, key_id: nil, level: 5)
345
464
  version_check
346
465
  fingerprint = key_id || fingerprint(email: email)
347
466
  return unless fingerprint
348
467
 
349
- command = "#{executable} --import-ownertrust"
468
+ command = gpg_command("--import-ownertrust")
350
469
  trust = "#{fingerprint}:#{level + 1}:\n"
351
- out, err, status = Open3.capture3(command, stdin_data: trust)
352
- logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
470
+ out, err, status = Open3.capture3(*command, stdin_data: trust)
471
+ IOStreams.logger&.debug { "IOStreams::Pgp.set_trust: #{command.shelljoin}\n#{err}#{out}" }
353
472
 
354
473
  raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
355
474
 
356
475
  err
357
476
  end
358
477
 
359
- # DEPRECATED - Use key_ids instead of fingerprints
478
+ # Internal: resolve an email address to a key fingerprint.
479
+ # Public callers should identify keys by `key_id` (see #list_keys / #key_info).
360
480
  def self.fingerprint(email:)
361
481
  version_check
362
- Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
482
+ command = gpg_command("--list-keys", "--fingerprint", "--with-colons", email.to_s)
483
+ Open3.popen2e(*command) do |_stdin, out, waith_thr|
363
484
  output = out.read.chomp
364
- if !waith_thr.value.success? && !(output !~ /(public key not found|No public key)/i)
485
+ if !waith_thr.value.success? && output !~ /(public key not found|No public key)/i
365
486
  raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
366
487
  end
367
488
 
368
489
  output.each_line do |line|
369
- if (match = line.match(/\Afpr.*::([^\:]*):\Z/))
490
+ if (match = line.match(/\Afpr.*::([^:]*):\Z/))
370
491
  return match[1]
371
492
  end
372
493
  end
373
494
  nil
374
495
  end
375
496
  end
376
-
377
- def self.logger=(logger)
378
- @logger = logger
379
- end
497
+ private_class_method :fingerprint
380
498
 
381
499
  # Returns [String] the version of pgp currently installed
382
500
  def self.pgp_version
383
501
  @pgp_version ||= begin
384
- command = "#{executable} --version"
385
- out, err, status = Open3.capture3(command)
386
- logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
502
+ command = gpg_command("--version")
503
+ out, err, status = Open3.capture3(*command)
504
+ IOStreams.logger&.debug { "IOStreams::Pgp.version: #{command.shelljoin}\n#{err}#{out}" }
387
505
  if status.success?
388
506
  # Sample output
389
507
  # #{executable} (GnuPG) 2.0.30
@@ -413,12 +531,6 @@ module IOStreams
413
531
  end
414
532
  end
415
533
 
416
- @logger = nil
417
-
418
- def self.logger
419
- @logger
420
- end
421
-
422
534
  def self.version_check
423
535
  # Previously, this method raised an error for versions >= 2.4
424
536
  # Now we support versions up to and including 2.4.7
@@ -507,6 +619,13 @@ module IOStreams
507
619
  results
508
620
  end
509
621
 
622
+ def self.reject_newlines!(**fields)
623
+ fields.each_pair do |field, value|
624
+ next if value.nil?
625
+ raise(ArgumentError, "IOStreams::Pgp.generate_key: :#{field} cannot contain newlines") if value.to_s =~ /[\r\n]/
626
+ end
627
+ end
628
+
510
629
  def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
511
630
  keys = private ? "secret-keys" : "keys"
512
631
 
@@ -517,9 +636,9 @@ module IOStreams
517
636
  key_id = key_info[:key_id]
518
637
  next unless key_id
519
638
 
520
- command = "#{executable} --batch --no-tty --yes --delete-#{keys} #{key_id}"
521
- out, err, status = Open3.capture3(command, binmode: true)
522
- logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
639
+ command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", key_id)
640
+ out, err, status = Open3.capture3(*command, binmode: true)
641
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}#{out}" }
523
642
 
524
643
  unless status.success?
525
644
  raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
@@ -532,18 +651,27 @@ module IOStreams
532
651
  def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
533
652
  keys = private ? "secret-keys" : "keys"
534
653
 
535
- command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email || key_id} | grep \"^fpr\" | cut -d: -f10`; do\n"
536
- command << "#{executable} --batch --no-tty --yes --delete-#{keys} \"$i\" ;\n"
537
- command << "done"
654
+ # List the fingerprints, then delete each one. Previously this shelled out
655
+ # to a `for` loop, which allowed shell injection via :email / :key_id.
656
+ list_command = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", (email || key_id).to_s)
657
+ list_out, list_err, = Open3.capture3(*list_command, binmode: true)
658
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{list_command.shelljoin}\n#{list_err}: #{list_out}" }
659
+
660
+ return false if list_err =~ /(not found|no public key)/i
661
+
662
+ fingerprints = list_out.each_line.select { |line| line.start_with?("fpr") }.map { |line| line.split(":")[9] }.compact
663
+ return false if fingerprints.empty?
538
664
 
539
- out, err, status = Open3.capture3(command, binmode: true)
540
- logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" }
665
+ fingerprints.each do |fingerprint|
666
+ command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", fingerprint)
667
+ out, err, status = Open3.capture3(*command, binmode: true)
668
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}: #{out}" }
541
669
 
542
- return false if err =~ /(not found|no public key)/i
543
- unless status.success?
544
- raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
670
+ unless status.success?
671
+ raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
672
+ end
673
+ raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
545
674
  end
546
- raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
547
675
 
548
676
  true
549
677
  end
@@ -10,13 +10,13 @@ module IOStreams
10
10
  end
11
11
 
12
12
  # When a Writer supports streams, also allow it to simply support a file
13
- def self.file(file_name, original_file_name: file_name, **args, &block)
14
- ::File.open(file_name, "rb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
13
+ def self.file(file_name, **args, &block)
14
+ ::File.open(file_name, "rb") { |file| stream(file, **args, &block) }
15
15
  end
16
16
 
17
17
  # For processing by either a file name or an open IO stream.
18
- def self.open(file_name_or_io, **args, &block)
19
- file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
18
+ def self.open(file_name_or_io, **args, &)
19
+ file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
20
20
  end
21
21
 
22
22
  attr_reader :input_stream
@@ -16,7 +16,7 @@ module IOStreams
16
16
 
17
17
  # When reading from a file also add the line reader stream
18
18
  def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
19
- IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
19
+ IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
20
20
  yield new(io, original_file_name: original_file_name, **args)
21
21
  end
22
22
  end
@@ -35,11 +35,10 @@ module IOStreams
35
35
  # format_options: [Hash]
36
36
  # Any specialized format specific options. For example, `:fixed` format requires the file definition.
37
37
  #
38
- # columns [Array<String>]
38
+ # columns [Array<String|Symbol>]
39
39
  # The header columns when the file does not include a header row.
40
40
  # Note:
41
- # It is recommended to keep all columns as strings to avoid any issues when persistence
42
- # with MongoDB when it converts symbol keys to strings.
41
+ # Column names are converted to strings.
43
42
  #
44
43
  # allowed_columns [Array<String>]
45
44
  # List of columns to allow.
@@ -18,7 +18,7 @@ module IOStreams
18
18
 
19
19
  # When writing to a file also add the line writer stream
20
20
  def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
21
- IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
21
+ IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
22
22
  yield new(io, original_file_name: original_file_name, **args, &block)
23
23
  end
24
24
  end
@@ -37,11 +37,10 @@ module IOStreams
37
37
  # format_options: [Hash]
38
38
  # Any specialized format specific options. For example, `:fixed` format requires the file definition.
39
39
  #
40
- # columns [Array<String>]
40
+ # columns [Array<String|Symbol>]
41
41
  # The header columns when the file does not include a header row.
42
42
  # Note:
43
- # It is recommended to keep all columns as strings to avoid any issues when persistence
44
- # with MongoDB when it converts symbol keys to strings.
43
+ # Column names are converted to strings.
45
44
  #
46
45
  # allowed_columns [Array<String>]
47
46
  # List of columns to allow.
@@ -14,7 +14,7 @@ module IOStreams
14
14
 
15
15
  # When reading from a file also add the line reader stream
16
16
  def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
17
- IOStreams::Line::Reader.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
17
+ IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
18
18
  yield new(io, original_file_name: original_file_name, **args)
19
19
  end
20
20
  end
@@ -21,7 +21,7 @@ module IOStreams
21
21
 
22
22
  # When writing to a file also add the line writer stream
23
23
  def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
24
- IOStreams::Line::Writer.file(file_name, original_file_name: original_file_name, delimiter: delimiter) do |io|
24
+ IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
25
25
  yield new(io, original_file_name: original_file_name, **args, &block)
26
26
  end
27
27
  end
@@ -18,8 +18,8 @@ module IOStreams
18
18
  # Example:
19
19
  #
20
20
  # IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
21
- def stream(stream, **options)
22
- builder.stream(stream, **options)
21
+ def stream(stream, **)
22
+ builder.stream(stream, **)
23
23
  self
24
24
  end
25
25
 
@@ -33,15 +33,15 @@ module IOStreams
33
33
  # IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
34
34
  #
35
35
  # IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
36
- def option(stream, **options)
37
- builder.option(stream, **options)
36
+ def option(stream, **)
37
+ builder.option(stream, **)
38
38
  self
39
39
  end
40
40
 
41
41
  # Adds the options for the specified stream as an option,
42
42
  # but if streams have already been added it is instead added as a stream.
43
- def option_or_stream(stream, **options)
44
- builder.option_or_stream(stream, **options)
43
+ def option_or_stream(stream, **)
44
+ builder.option_or_stream(stream, **)
45
45
  self
46
46
  end
47
47
 
@@ -93,21 +93,26 @@ module IOStreams
93
93
  def each(mode = :line, **args, &block)
94
94
  raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
95
95
 
96
- # return enum_for __method__ unless block_given?
96
+ # Deliberately not returning an Enumerator when no block is given.
97
+ # The stream pipeline manages resources via block scope: every stream is opened with an
98
+ # `ensure` that closes the file handle, reaps the gpg subprocess, deletes temp files, etc.
99
+ # A Fiber-backed Enumerator (e.g. `to_enum(__method__, mode, **args)`) would leave that block
100
+ # suspended; if the caller abandons a partially-consumed enumerator, none of the cleanup runs
101
+ # until GC collects the Fiber, leaking file descriptors, gpg processes, and temp files.
97
102
  reader(mode, **args) { |stream| stream.each(&block) }
98
103
  end
99
104
 
100
105
  # Returns a Reader for reading a file / stream
101
- def reader(mode = :stream, **args, &block)
106
+ def reader(mode = :stream, **args, &)
102
107
  case mode
103
108
  when :stream
104
- stream_reader(&block)
109
+ stream_reader(&)
105
110
  when :line
106
- line_reader(**args, &block)
111
+ line_reader(**args, &)
107
112
  when :array
108
- row_reader(**args, &block)
113
+ row_reader(**args, &)
109
114
  when :hash
110
- record_reader(**args, &block)
115
+ record_reader(**args, &)
111
116
  else
112
117
  raise(ArgumentError, "Invalid mode: #{mode.inspect}")
113
118
  end
@@ -124,16 +129,16 @@ module IOStreams
124
129
  end
125
130
 
126
131
  # Returns a Writer for writing to a file / stream
127
- def writer(mode = :stream, **args, &block)
132
+ def writer(mode = :stream, **args, &)
128
133
  case mode
129
134
  when :stream
130
- stream_writer(&block)
135
+ stream_writer(&)
131
136
  when :line
132
- line_writer(**args, &block)
137
+ line_writer(**args, &)
133
138
  when :array
134
- row_writer(**args, &block)
139
+ row_writer(**args, &)
135
140
  when :hash
136
- record_writer(**args, &block)
141
+ record_writer(**args, &)
137
142
  else
138
143
  raise(ArgumentError, "Invalid mode: #{mode.inspect}")
139
144
  end
@@ -171,7 +176,7 @@ module IOStreams
171
176
  # IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", convert: false)
172
177
  #
173
178
  # # Advanced copy with custom stream conversions on source and target.
174
- # source = IOStreams.path("source_file").stream(encoding: "BINARY")
179
+ # source = IOStreams.path("source_file").stream(:encode, encoding: "BINARY")
175
180
  # IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
176
181
  def copy_from(source, convert: true, mode: nil, **args)
177
182
  if convert
@@ -322,18 +327,19 @@ module IOStreams
322
327
  @builder ||= IOStreams::Builder.new
323
328
  end
324
329
 
325
- def stream_reader(&block)
326
- builder.reader(io_stream, &block)
330
+ def stream_reader(&)
331
+ builder.reader(io_stream, &)
327
332
  end
328
333
 
329
334
  def line_reader(embedded_within: nil, **args)
330
335
  embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
331
336
 
332
337
  stream_reader do |io|
333
- yield IOStreams::Line::Reader.new(io,
334
- original_file_name: builder.file_name,
335
- embedded_within: embedded_within,
336
- **args)
338
+ yield IOStreams::Line::Reader.new(
339
+ io,
340
+ embedded_within: embedded_within,
341
+ **args
342
+ )
337
343
  end
338
344
  end
339
345
 
@@ -363,20 +369,20 @@ module IOStreams
363
369
  end
364
370
  end
365
371
 
366
- def stream_writer(&block)
367
- builder.writer(io_stream, &block)
372
+ def stream_writer(&)
373
+ builder.writer(io_stream, &)
368
374
  end
369
375
 
370
376
  def line_writer(**args, &block)
371
- return block.call(io_stream) if io_stream&.is_a?(IOStreams::Line::Writer)
377
+ return block.call(io_stream) if io_stream.is_a?(IOStreams::Line::Writer)
372
378
 
373
379
  writer do |io|
374
- IOStreams::Line::Writer.stream(io, original_file_name: builder.file_name, **args, &block)
380
+ IOStreams::Line::Writer.stream(io, **args, &block)
375
381
  end
376
382
  end
377
383
 
378
384
  def row_writer(delimiter: $/, **args, &block)
379
- return block.call(io_stream) if io_stream&.is_a?(IOStreams::Row::Writer)
385
+ return block.call(io_stream) if io_stream.is_a?(IOStreams::Row::Writer)
380
386
 
381
387
  line_writer(delimiter: delimiter) do |io|
382
388
  IOStreams::Row::Writer.stream(
@@ -391,7 +397,7 @@ module IOStreams
391
397
  end
392
398
 
393
399
  def record_writer(delimiter: $/, **args, &block)
394
- return block.call(io_stream) if io_stream&.is_a?(IOStreams::Record::Writer)
400
+ return block.call(io_stream) if io_stream.is_a?(IOStreams::Record::Writer)
395
401
 
396
402
  line_writer(delimiter: delimiter) do |io|
397
403
  IOStreams::Record::Writer.stream(
@@ -2,10 +2,10 @@ module IOStreams
2
2
  module SymmetricEncryption
3
3
  class Reader < IOStreams::Reader
4
4
  # read from a file/stream using Symmetric Encryption
5
- def self.stream(input_stream, **args, &block)
5
+ def self.stream(input_stream, **args, &)
6
6
  Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
7
7
 
8
- ::SymmetricEncryption::Reader.open(input_stream, **args, &block)
8
+ ::SymmetricEncryption::Reader.open(input_stream, **args, &)
9
9
  end
10
10
  end
11
11
  end