iostreams 1.11.0 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (108) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +14 -13
  3. data/Rakefile +52 -0
  4. data/docs/CLAUDE.md +9 -0
  5. data/docs/config.md +157 -0
  6. data/docs/copy_files.md +75 -0
  7. data/docs/extensions.md +111 -0
  8. data/docs/formats.md +188 -0
  9. data/docs/index.md +388 -0
  10. data/docs/path.md +652 -0
  11. data/docs/pgp.md +436 -0
  12. data/docs/streams.md +337 -0
  13. data/docs/tutorial.md +483 -0
  14. data/docs/upgrading.md +217 -0
  15. data/lib/io_streams/builder.rb +71 -11
  16. data/lib/io_streams/bzip2/reader.rb +25 -2
  17. data/lib/io_streams/bzip2/writer.rb +26 -2
  18. data/lib/io_streams/encode/reader.rb +6 -2
  19. data/lib/io_streams/encode/writer.rb +9 -5
  20. data/lib/io_streams/errors.rb +4 -0
  21. data/lib/io_streams/gzip/reader.rb +5 -1
  22. data/lib/io_streams/gzip/writer.rb +11 -2
  23. data/lib/io_streams/io_streams.rb +156 -20
  24. data/lib/io_streams/line/reader.rb +9 -4
  25. data/lib/io_streams/line/writer.rb +1 -1
  26. data/lib/io_streams/path.rb +117 -8
  27. data/lib/io_streams/paths/file.rb +57 -11
  28. data/lib/io_streams/paths/http.rb +123 -9
  29. data/lib/io_streams/paths/matcher.rb +3 -3
  30. data/lib/io_streams/paths/s3.rb +69 -18
  31. data/lib/io_streams/paths/sftp/net_ssh.rb +104 -0
  32. data/lib/io_streams/paths/sftp.rb +103 -64
  33. data/lib/io_streams/pgp/reader.rb +63 -10
  34. data/lib/io_streams/pgp/writer.rb +111 -30
  35. data/lib/io_streams/pgp.rb +256 -71
  36. data/lib/io_streams/reader.rb +14 -5
  37. data/lib/io_streams/record/reader.rb +75 -6
  38. data/lib/io_streams/record/writer.rb +3 -4
  39. data/lib/io_streams/row/reader.rb +1 -1
  40. data/lib/io_streams/row/writer.rb +1 -1
  41. data/lib/io_streams/stream.rb +48 -37
  42. data/lib/io_streams/symmetric_encryption/reader.rb +6 -2
  43. data/lib/io_streams/symmetric_encryption/writer.rb +8 -4
  44. data/lib/io_streams/tabular/header.rb +49 -10
  45. data/lib/io_streams/tabular/parser/array.rb +0 -10
  46. data/lib/io_streams/tabular/parser/base.rb +10 -0
  47. data/lib/io_streams/tabular/parser/csv.rb +9 -36
  48. data/lib/io_streams/tabular/parser/fixed.rb +8 -6
  49. data/lib/io_streams/tabular/parser/psv.rb +6 -14
  50. data/lib/io_streams/tabular.rb +5 -10
  51. data/lib/io_streams/utils.rb +34 -2
  52. data/lib/io_streams/version.rb +1 -1
  53. data/lib/io_streams/writer.rb +16 -7
  54. data/lib/io_streams/xlsx/reader.rb +6 -2
  55. data/lib/io_streams/zip/reader.rb +4 -0
  56. data/lib/io_streams/zip/writer.rb +26 -10
  57. data/lib/iostreams.rb +0 -1
  58. metadata +46 -112
  59. data/lib/io_streams/deprecated.rb +0 -216
  60. data/lib/io_streams/tabular/utility/csv_row.rb +0 -105
  61. data/test/builder_test.rb +0 -311
  62. data/test/bzip2_reader_test.rb +0 -27
  63. data/test/bzip2_writer_test.rb +0 -56
  64. data/test/deprecated_test.rb +0 -121
  65. data/test/encode_reader_test.rb +0 -51
  66. data/test/encode_writer_test.rb +0 -90
  67. data/test/files/embedded_lines_test.csv +0 -7
  68. data/test/files/multiple_files.zip +0 -0
  69. data/test/files/spreadsheet.xlsx +0 -0
  70. data/test/files/test.csv +0 -4
  71. data/test/files/test.json +0 -3
  72. data/test/files/test.psv +0 -4
  73. data/test/files/text file.txt +0 -3
  74. data/test/files/text.txt +0 -3
  75. data/test/files/text.txt.bz2 +0 -0
  76. data/test/files/text.txt.gz +0 -0
  77. data/test/files/text.txt.gz.zip +0 -0
  78. data/test/files/text.zip +0 -0
  79. data/test/files/text.zip.gz +0 -0
  80. data/test/files/unclosed_quote_large_test.csv +0 -1658
  81. data/test/files/unclosed_quote_test.csv +0 -4
  82. data/test/files/unclosed_quote_test2.csv +0 -3
  83. data/test/gzip_reader_test.rb +0 -27
  84. data/test/gzip_writer_test.rb +0 -52
  85. data/test/io_streams_test.rb +0 -132
  86. data/test/line_reader_test.rb +0 -325
  87. data/test/line_writer_test.rb +0 -59
  88. data/test/minimal_file_reader.rb +0 -25
  89. data/test/path_test.rb +0 -55
  90. data/test/paths/file_test.rb +0 -213
  91. data/test/paths/http_test.rb +0 -34
  92. data/test/paths/matcher_test.rb +0 -120
  93. data/test/paths/s3_test.rb +0 -220
  94. data/test/paths/sftp_test.rb +0 -106
  95. data/test/pgp_reader_test.rb +0 -46
  96. data/test/pgp_test.rb +0 -267
  97. data/test/pgp_writer_test.rb +0 -130
  98. data/test/record_reader_test.rb +0 -60
  99. data/test/record_writer_test.rb +0 -82
  100. data/test/row_reader_test.rb +0 -35
  101. data/test/row_writer_test.rb +0 -56
  102. data/test/stream_test.rb +0 -577
  103. data/test/tabular_test.rb +0 -338
  104. data/test/test_helper.rb +0 -40
  105. data/test/utils_test.rb +0 -20
  106. data/test/xlsx_reader_test.rb +0 -37
  107. data/test/zip_reader_test.rb +0 -53
  108. data/test/zip_writer_test.rb +0 -48
@@ -1,4 +1,5 @@
1
1
  require "open3"
2
+ require "shellwords"
2
3
  module IOStreams
3
4
  # Read/Write PGP/GPG file or stream.
4
5
  #
@@ -25,6 +26,18 @@ module IOStreams
25
26
 
26
27
  @executable = "gpg"
27
28
 
29
+ # Returns [Array<String>] the argv used to invoke gpg, with the supplied
30
+ # arguments appended.
31
+ #
32
+ # All gpg invocations are run without a shell (the multi-argument form of
33
+ # `Open3`) so that values such as email addresses, key ids, passphrases and
34
+ # file names cannot be interpreted as shell commands. The configured
35
+ # `executable` is split with `Shellwords` so that it may still contain
36
+ # additional fixed arguments (for example "gpg --homedir /path").
37
+ def self.gpg_command(*args)
38
+ Shellwords.split(executable) + args.map(&:to_s)
39
+ end
40
+
28
41
  # Generate a new ultimate trusted local public and private key.
29
42
  #
30
43
  # Returns [String] the key id for the generated key.
@@ -44,6 +57,20 @@ module IOStreams
44
57
  # Highly Recommended.
45
58
  # To generate a good passphrase:
46
59
  # `SecureRandom.urlsafe_base64(128)`
60
+ # Pass `nil` to generate an unprotected (passphrase-less) key.
61
+ #
62
+ # key_curve / subkey_curve [String]
63
+ # Optional Elliptic Curve to use for the (sub)key, e.g. "ed25519".
64
+ # When supplied the corresponding key/subkey length is ignored.
65
+ # Requires GnuPG 2.1 or later.
66
+ #
67
+ # key_usage / subkey_usage [String]
68
+ # Optional comma separated list of (sub)key capabilities, e.g. "sign".
69
+ # Requires GnuPG 2.1 or later.
70
+ #
71
+ # creation_date [String]
72
+ # Optional creation date for the key, e.g. "20240101T000000".
73
+ # Requires GnuPG 2.1 or later.
47
74
  #
48
75
  # See `man gpg` for the remaining options
49
76
  def self.generate_key(name:,
@@ -54,23 +81,70 @@ module IOStreams
54
81
  key_length: 4096,
55
82
  subkey_type: "RSA",
56
83
  subkey_length: key_length,
84
+ key_curve: nil,
85
+ key_usage: nil,
86
+ subkey_curve: nil,
87
+ subkey_usage: nil,
88
+ creation_date: nil,
57
89
  expire_date: nil)
58
90
  version_check
59
- params = ""
91
+
92
+ # Reject newlines so that a value cannot inject additional directives into
93
+ # the gpg batch key-generation parameter file.
94
+ reject_newlines!(name: name, email: email, comment: comment, passphrase: passphrase,
95
+ key_type: key_type, subkey_type: subkey_type, expire_date: expire_date,
96
+ key_curve: key_curve, key_usage: key_usage,
97
+ subkey_curve: subkey_curve, subkey_usage: subkey_usage,
98
+ creation_date: creation_date)
99
+
100
+ # `%no-protection`, and the Elliptic Curve / usage / creation-date directives
101
+ # were all introduced in GnuPG 2.1. Keep older versions working by only
102
+ # emitting them when a 2.1+ binary is detected. `--batch --gen-key` accepts
103
+ # all of these on 2.1+, so there is no need for the newer `--full-gen-key`.
104
+ modern = pgp_version.to_f >= 2.1
105
+
106
+ unless modern
107
+ new_options = {
108
+ key_curve: key_curve,
109
+ key_usage: key_usage,
110
+ subkey_curve: subkey_curve,
111
+ subkey_usage: subkey_usage,
112
+ creation_date: creation_date
113
+ }.compact
114
+ unless new_options.empty?
115
+ raise(ArgumentError,
116
+ "IOStreams::Pgp.generate_key: #{new_options.keys.join(', ')} require GnuPG 2.1 or later " \
117
+ "(detected #{pgp_version})")
118
+ end
119
+ end
120
+
121
+ params = +""
122
+ # `%no-protection` is a control statement and must precede the key parameters.
123
+ # GnuPG 2.1+ requires this explicit opt-out to create an unprotected key;
124
+ # older versions create one simply by omitting the Passphrase directive.
125
+ params << "%no-protection\n" if !passphrase && modern
60
126
  params << "Key-Type: #{key_type}\n" if key_type
61
- params << "Key-Length: #{key_length}\n" if key_length
127
+ # Key-Length and Key-Curve are mutually exclusive: curves imply their own length.
128
+ params << "Key-Length: #{key_length}\n" if key_length && !key_curve
129
+ params << "Key-Curve: #{key_curve}\n" if key_curve
130
+ params << "Key-Usage: #{key_usage}\n" if key_usage
62
131
  params << "Subkey-Type: #{subkey_type}\n" if subkey_type
63
- params << "Subkey-Length: #{subkey_length}\n" if subkey_length
132
+ params << "Subkey-Length: #{subkey_length}\n" if subkey_length && !subkey_curve
133
+ params << "Subkey-Curve: #{subkey_curve}\n" if subkey_curve
134
+ params << "Subkey-Usage: #{subkey_usage}\n" if subkey_usage
64
135
  params << "Name-Real: #{name}\n" if name
65
136
  params << "Name-Comment: #{comment}\n" if comment
66
137
  params << "Name-Email: #{email}\n" if email
67
138
  params << "Expire-Date: #{expire_date}\n" if expire_date
139
+ params << "Creation-Date: #{creation_date}\n" if creation_date
68
140
  params << "Passphrase: #{passphrase}\n" if passphrase
69
141
  params << "%commit"
70
- command = "#{executable} --batch --gen-key --no-tty"
71
142
 
72
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: params)
73
- logger&.debug { "IOStreams::Pgp.generate_key: #{command}\n#{params}\n#{err}#{out}" }
143
+ command = gpg_command("--batch", "--gen-key", "--no-tty")
144
+
145
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: params)
146
+ # Do not log `params`, it contains the passphrase.
147
+ IOStreams.logger&.debug { "IOStreams::Pgp.generate_key: #{command.shelljoin}\n#{err}#{out}" }
74
148
 
75
149
  raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
76
150
 
@@ -86,13 +160,14 @@ module IOStreams
86
160
  end
87
161
  end
88
162
 
89
- # Delete all private and public keys for a particular email.
163
+ # Delete all private and public keys for a particular email or key id.
90
164
  #
91
165
  # Returns false if no key was found.
92
166
  # Raises an exception if it fails to delete the key.
167
+ # Raises ArgumentError when neither :email nor :key_id is supplied.
93
168
  #
94
- # email: [String] Optional email address for the key.
95
- # key_id: [String] Optional id for the key.
169
+ # email: [String] Email address for the key.
170
+ # key_id: [String] Id for the key.
96
171
  #
97
172
  # public: [true|false]
98
173
  # Whether to delete the public key
@@ -102,6 +177,8 @@ module IOStreams
102
177
  # Whether to delete the private key
103
178
  # Default: false
104
179
  def self.delete_keys(email: nil, key_id: nil, public: true, private: false)
180
+ raise(ArgumentError, "Either :email, or :key_id must be supplied") if email.nil? && key_id.nil?
181
+
105
182
  version_check
106
183
  # Version 2.1+ uses delete_public_or_private_keys
107
184
  # Version < 2.1 uses delete_public_or_private_keys_v1
@@ -127,14 +204,18 @@ module IOStreams
127
204
  # date: [String]
128
205
  # name: [String]
129
206
  # email: [String]
207
+ # private: [true|false]
208
+ # trust: [String]
130
209
  # Returns [] if no keys were found.
131
210
  def self.list_keys(email: nil, key_id: nil, private: false)
132
211
  version_check
133
- cmd = private ? "--list-secret-keys" : "--list-keys"
134
- command = "#{executable} #{cmd} #{email || key_id}"
212
+ args = [private ? "--list-secret-keys" : "--list-keys"]
213
+ # `--` stops gpg from treating the email or key id as an option.
214
+ args += ["--", (email || key_id).to_s] if email || key_id
215
+ command = gpg_command(*args)
135
216
 
136
- out, err, status = Open3.capture3(command, binmode: true)
137
- logger&.debug { "IOStreams::Pgp.list_keys: #{command}\n#{err}#{out}" }
217
+ out, err, status = Open3.capture3(*command, binmode: true)
218
+ IOStreams.logger&.debug { "IOStreams::Pgp.list_keys: #{command.shelljoin}\n#{err}#{out}" }
138
219
  if status.success? && out.length.positive?
139
220
  parse_list_output(out)
140
221
  else
@@ -156,12 +237,14 @@ module IOStreams
156
237
  # date: [String]
157
238
  # name: [String]
158
239
  # email: [String]
240
+ # private: [true|false]
241
+ # trust: [String]
159
242
  def self.key_info(key:)
160
243
  version_check
161
- command = "#{executable} --batch --no-tty"
244
+ command = gpg_command("--batch", "--no-tty")
162
245
 
163
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
164
- logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
246
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
247
+ IOStreams.logger&.debug { "IOStreams::Pgp.key_info: #{command.shelljoin}\n#{err}#{out}" }
165
248
 
166
249
  # Try parsing even if we get an error - some versions of GPG return non-zero status but still output key info
167
250
  unless (status.success? || err.include?("key ID") || out.include?("pub")) && out.length.positive?
@@ -176,28 +259,43 @@ module IOStreams
176
259
  parse_list_output(out)
177
260
  end
178
261
 
179
- # Returns [String] containing all the public keys for the supplied email address.
262
+ # Returns [String] containing all the public keys for the supplied email address or key id.
263
+ #
264
+ # Raises ArgumentError when neither :email nor :key_id is supplied.
265
+ # Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added.
266
+ # In v3.0 this will raise ArgumentError instead.
180
267
  #
181
268
  # email: [String] Email address for requested key.
182
269
  #
270
+ # key_id: [String] Id for the requested key.
271
+ #
183
272
  # ascii: [true|false]
184
273
  # Whether to export as ASCII text instead of binary format
185
274
  # Default: true
186
- def self.export(email:, ascii: true, private: false, passphrase: nil)
275
+ def self.export(email: (email_omitted = true) && nil, key_id: nil, ascii: true, private: false, passphrase: nil)
276
+ if email.nil? && key_id.nil?
277
+ raise(ArgumentError, "Either :email, or :key_id must be supplied") if email_omitted
278
+
279
+ raise(Pgp::Failure, "GPG Failed reading key: no email address or key id was supplied")
280
+ end
281
+
187
282
  version_check
188
283
 
189
- command = "#{executable} "
190
- command << "--pinentry-mode loopback " if pgp_version.to_f >= 2.1
191
- command << "--no-symkey-cache " if pgp_version.to_f >= 2.4
192
- command << "--armor " if ascii
193
- command << "--no-tty --batch --passphrase"
194
- command << (passphrase ? " #{passphrase} " : "-fd 0 ")
195
- command << (private ? "--export-secret-keys #{email}" : "--export #{email}")
284
+ args = []
285
+ args += ["--pinentry-mode", "loopback"] if pgp_version.to_f >= 2.1
286
+ args << "--no-symkey-cache" if pgp_version.to_f >= 2.4
287
+ args << "--armor" if ascii
288
+ args += ["--no-tty", "--batch"]
289
+ # Supply the passphrase on stdin so that it is not visible in the process list.
290
+ args += ["--passphrase-fd", "0"]
291
+ args << (private ? "--export-secret-keys" : "--export")
292
+ args += ["--", (email || key_id).to_s]
293
+ command = gpg_command(*args)
196
294
 
197
- out, err, status = Open3.capture3(command, binmode: true)
198
- logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
295
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: passphrase.to_s)
296
+ IOStreams.logger&.debug { "IOStreams::Pgp.export: #{command.shelljoin}\n#{err}" }
199
297
 
200
- raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
298
+ raise(Pgp::Failure, "GPG Failed reading key: #{email || key_id}: #{err}") unless status.success? && out.length.positive?
201
299
 
202
300
  out
203
301
  end
@@ -219,10 +317,10 @@ module IOStreams
219
317
  # * Invalidated keys must be removed manually.
220
318
  def self.import(key:)
221
319
  version_check
222
- command = "#{executable} --batch --import"
320
+ command = gpg_command("--batch", "--import")
223
321
 
224
- out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
225
- logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
322
+ out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
323
+ IOStreams.logger&.debug { "IOStreams::Pgp.import: #{command.shelljoin}\n#{err}#{out}" }
226
324
 
227
325
  # Handle both old and new versions of GPG
228
326
  # For older versions, the output is in err, for newer ones it might be in out
@@ -316,23 +414,74 @@ module IOStreams
316
414
  raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
317
415
  end
318
416
 
319
- # Returns [String] email for the supplied after importing and trusting the key
417
+ # Imports the supplied key and then marks it as trusted at the supplied trust level.
418
+ #
419
+ # Returns [String] email for the supplied key, or its key id when no email is present.
420
+ #
421
+ # key: [String]
422
+ # The public (or private) key to import and trust.
423
+ #
424
+ # trust_level: [Integer]
425
+ # The owner-trust level to assign to the imported key, the same levels used by `set_trust`:
426
+ # 1 : Undefined (no opinion)
427
+ # 2 : Never (do not trust)
428
+ # 3 : Marginal
429
+ # 4 : Full
430
+ # 5 : Ultimate
431
+ # Default: 5 : Ultimate
432
+ #
433
+ # SECURITY WARNING:
434
+ # Only import and trust keys received from a verified, trusted source.
435
+ # The default trust level is `5` (Ultimate), which tells GPG to treat the imported key
436
+ # as if it were one of your own keys. An ultimately trusted key is implicitly valid and
437
+ # can in turn confer validity on other keys it has signed. Importing an attacker supplied
438
+ # key at this level allows that attacker to impersonate other recipients.
439
+ # When the key cannot be fully verified, supply a lower `trust_level`.
320
440
  #
321
441
  # Notes:
322
442
  # - If the same email address has multiple keys then only the first is currently trusted.
323
- def self.import_and_trust(key:)
443
+ def self.import_and_trust(key:, trust_level: 5)
444
+ info = import_and_trust_key_info(key: key, trust_level: trust_level)
445
+ info[:email] || info[:key_id]
446
+ end
447
+
448
+ # Imports and trusts the supplied key, see #import_and_trust.
449
+ #
450
+ # Returns [String] the recipient to encrypt to: the key's fingerprint when gpg supplies it,
451
+ # otherwise the email address or key id returned by #import_and_trust.
452
+ #
453
+ # Encrypting to the fingerprint ensures that the imported key is used, since gpg looks up an
454
+ # email address in the keyring, where another key may have the same email address.
455
+ #
456
+ # Used internally by the PGP writer.
457
+ def self.import_and_trust_recipient(key:, trust_level: 5)
458
+ info = import_and_trust_key_info(key: key, trust_level: trust_level)
459
+ key_id = info[:key_id].to_s
460
+ # gpg v2.1 and later supply the full fingerprint: 40 hex digits for v4 keys, 64 for v5 and v6 keys.
461
+ # Earlier versions only supply a short key id, which is not unique.
462
+ return key_id if key_id.match?(/\A(\h{40}|\h{64})\z/)
463
+
464
+ info[:email] || info[:key_id]
465
+ end
466
+
467
+ # Returns [Hash] the key info for the supplied key, after importing and trusting it.
468
+ def self.import_and_trust_key_info(key:, trust_level:)
324
469
  raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
325
470
 
326
- key_info = key_info(key: key).last
471
+ infos = key_info(key: key)
472
+ info = infos.last&.dup || {}
473
+ # When a key has several user ids, only the entry for its first user id includes the key id.
474
+ info[:key_id] ||= infos.reverse_each.find { |entry| entry[:key_id] }&.fetch(:key_id)
327
475
 
328
- email = key_info.fetch(:email, nil)
329
- key_id = key_info.fetch(:key_id, nil)
476
+ email = info[:email]
477
+ key_id = info[:key_id]
330
478
  raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
331
479
 
332
480
  import(key: key)
333
- set_trust(email: email, key_id: key_id)
334
- email || key_id
481
+ set_trust(email: email, key_id: key_id, level: trust_level)
482
+ info
335
483
  end
484
+ private_class_method :import_and_trust_key_info
336
485
 
337
486
  # Set the trust level for an existing key.
338
487
  #
@@ -340,50 +489,75 @@ module IOStreams
340
489
  # Returns nil if the email was not found
341
490
  #
342
491
  # After importing keys, they are not trusted and the relevant trust level must be set.
492
+ #
493
+ # key_id: [String]
494
+ # The fingerprint of the key, as hexadecimal digits only.
495
+ # Raises ArgumentError when it contains any other characters.
496
+ #
497
+ # level: [Integer]
498
+ # The owner-trust level to assign to the key:
499
+ # 1 : Undefined (no opinion)
500
+ # 2 : Never (do not trust)
501
+ # 3 : Marginal
502
+ # 4 : Full
503
+ # 5 : Ultimate
343
504
  # Default: 5 : Ultimate
505
+ #
506
+ # SECURITY WARNING:
507
+ # Only trust keys received from a verified, trusted source.
508
+ # The default trust level is `5` (Ultimate), which tells GPG to treat the key
509
+ # as if it were one of your own keys. An ultimately trusted key is implicitly valid and
510
+ # can in turn confer validity on other keys it has signed. Trusting an attacker supplied
511
+ # key at this level allows that attacker to impersonate other recipients.
512
+ # When the key cannot be fully verified, supply a lower `level`.
344
513
  def self.set_trust(email: nil, key_id: nil, level: 5)
514
+ # The key_id is written into gpg's ownertrust input, where any other character, such as a newline,
515
+ # could add trust lines for other keys.
516
+ if key_id && !key_id.to_s.match?(/\A\h+\z/)
517
+ raise(ArgumentError, "Invalid :key_id, it must only contain hexadecimal digits: #{key_id.inspect}")
518
+ end
519
+
345
520
  version_check
346
521
  fingerprint = key_id || fingerprint(email: email)
347
522
  return unless fingerprint
348
523
 
349
- command = "#{executable} --import-ownertrust"
524
+ command = gpg_command("--import-ownertrust")
350
525
  trust = "#{fingerprint}:#{level + 1}:\n"
351
- out, err, status = Open3.capture3(command, stdin_data: trust)
352
- logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
526
+ out, err, status = Open3.capture3(*command, stdin_data: trust)
527
+ IOStreams.logger&.debug { "IOStreams::Pgp.set_trust: #{command.shelljoin}\n#{err}#{out}" }
353
528
 
354
529
  raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
355
530
 
356
531
  err
357
532
  end
358
533
 
359
- # DEPRECATED - Use key_ids instead of fingerprints
534
+ # Internal: resolve an email address to a key fingerprint.
535
+ # Public callers should identify keys by `key_id` (see #list_keys / #key_info).
360
536
  def self.fingerprint(email:)
361
537
  version_check
362
- Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
538
+ command = gpg_command("--list-keys", "--fingerprint", "--with-colons", "--", email.to_s)
539
+ Open3.popen2e(*command) do |_stdin, out, waith_thr|
363
540
  output = out.read.chomp
364
- if !waith_thr.value.success? && !(output !~ /(public key not found|No public key)/i)
541
+ if !waith_thr.value.success? && output !~ /(public key not found|No public key)/i
365
542
  raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
366
543
  end
367
544
 
368
545
  output.each_line do |line|
369
- if (match = line.match(/\Afpr.*::([^\:]*):\Z/))
546
+ if (match = line.match(/\Afpr.*::([^:]*):\Z/))
370
547
  return match[1]
371
548
  end
372
549
  end
373
550
  nil
374
551
  end
375
552
  end
376
-
377
- def self.logger=(logger)
378
- @logger = logger
379
- end
553
+ private_class_method :fingerprint
380
554
 
381
555
  # Returns [String] the version of pgp currently installed
382
556
  def self.pgp_version
383
557
  @pgp_version ||= begin
384
- command = "#{executable} --version"
385
- out, err, status = Open3.capture3(command)
386
- logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
558
+ command = gpg_command("--version")
559
+ out, err, status = Open3.capture3(*command)
560
+ IOStreams.logger&.debug { "IOStreams::Pgp.version: #{command.shelljoin}\n#{err}#{out}" }
387
561
  if status.success?
388
562
  # Sample output
389
563
  # #{executable} (GnuPG) 2.0.30
@@ -413,12 +587,6 @@ module IOStreams
413
587
  end
414
588
  end
415
589
 
416
- @logger = nil
417
-
418
- def self.logger
419
- @logger
420
- end
421
-
422
590
  def self.version_check
423
591
  # Previously, this method raised an error for versions >= 2.4
424
592
  # Now we support versions up to and including 2.4.7
@@ -497,16 +665,24 @@ module IOStreams
497
665
  hash[:trust] = match[2].to_s.strip if match[1]
498
666
  results << hash
499
667
  hash = {}
500
- elsif (match = line.match(/\s+([A-Z0-9]{16,40})/))
668
+ elsif (match = line.match(/\s+([A-Z0-9]{16,64})/))
501
669
  # v2.2/v2.4 key id on separate line:
502
670
  # 18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
503
671
  # Or shorter format: 7932AB23D7238F6B
672
+ # Or a 64 digit fingerprint for v5 and v6 keys.
504
673
  hash[:key_id] ||= match[1]
505
674
  end
506
675
  end
507
676
  results
508
677
  end
509
678
 
679
+ def self.reject_newlines!(**fields)
680
+ fields.each_pair do |field, value|
681
+ next if value.nil?
682
+ raise(ArgumentError, "IOStreams::Pgp.generate_key: :#{field} cannot contain newlines") if value.to_s =~ /[\r\n]/
683
+ end
684
+ end
685
+
510
686
  def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
511
687
  keys = private ? "secret-keys" : "keys"
512
688
 
@@ -517,9 +693,9 @@ module IOStreams
517
693
  key_id = key_info[:key_id]
518
694
  next unless key_id
519
695
 
520
- command = "#{executable} --batch --no-tty --yes --delete-#{keys} #{key_id}"
521
- out, err, status = Open3.capture3(command, binmode: true)
522
- logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
696
+ command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", key_id)
697
+ out, err, status = Open3.capture3(*command, binmode: true)
698
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}#{out}" }
523
699
 
524
700
  unless status.success?
525
701
  raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
@@ -532,18 +708,27 @@ module IOStreams
532
708
  def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
533
709
  keys = private ? "secret-keys" : "keys"
534
710
 
535
- command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email || key_id} | grep \"^fpr\" | cut -d: -f10`; do\n"
536
- command << "#{executable} --batch --no-tty --yes --delete-#{keys} \"$i\" ;\n"
537
- command << "done"
711
+ # List the fingerprints, then delete each one. Previously this shelled out
712
+ # to a `for` loop, which allowed shell injection via :email / :key_id.
713
+ list_command = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", "--", (email || key_id).to_s)
714
+ list_out, list_err, = Open3.capture3(*list_command, binmode: true)
715
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{list_command.shelljoin}\n#{list_err}: #{list_out}" }
716
+
717
+ return false if list_err =~ /(not found|no public key)/i
718
+
719
+ fingerprints = list_out.each_line.select { |line| line.start_with?("fpr") }.map { |line| line.split(":")[9] }.compact
720
+ return false if fingerprints.empty?
538
721
 
539
- out, err, status = Open3.capture3(command, binmode: true)
540
- logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" }
722
+ fingerprints.each do |fingerprint|
723
+ command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", fingerprint)
724
+ out, err, status = Open3.capture3(*command, binmode: true)
725
+ IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}: #{out}" }
541
726
 
542
- return false if err =~ /(not found|no public key)/i
543
- unless status.success?
544
- raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
727
+ unless status.success?
728
+ raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
729
+ end
730
+ raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
545
731
  end
546
- raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
547
732
 
548
733
  true
549
734
  end
@@ -1,22 +1,31 @@
1
1
  module IOStreams
2
2
  class Reader
3
+ # Returns [Array<Symbol>] the names of the options this reader accepts,
4
+ # or [nil] when the reader does not declare them.
5
+ #
6
+ # When declared, `IOStreams::Builder` rejects any other option before the reader is opened,
7
+ # naming the direction an option belongs to when it is only valid for the other direction.
8
+ def self.option_names
9
+ nil
10
+ end
11
+
3
12
  # When a Reader does not support streams, we copy the stream to a local temp file
4
13
  # and then pass that filename in for this reader.
5
14
  def self.stream(input_stream, **args, &block)
6
- Utils.temp_file_name("iostreams_reader") do |file_name|
15
+ Utils.private_temp_file("iostreams_reader") do |file_name|
7
16
  ::File.open(file_name, "wb") { |target| ::IO.copy_stream(input_stream, target) }
8
17
  file(file_name, **args, &block)
9
18
  end
10
19
  end
11
20
 
12
21
  # When a Writer supports streams, also allow it to simply support a file
13
- def self.file(file_name, original_file_name: file_name, **args, &block)
14
- ::File.open(file_name, "rb") { |file| stream(file, original_file_name: original_file_name, **args, &block) }
22
+ def self.file(file_name, **args, &block)
23
+ ::File.open(file_name, "rb") { |file| stream(file, **args, &block) }
15
24
  end
16
25
 
17
26
  # For processing by either a file name or an open IO stream.
18
- def self.open(file_name_or_io, **args, &block)
19
- file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &block) : stream(file_name_or_io, **args, &block)
27
+ def self.open(file_name_or_io, **args, &)
28
+ file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
20
29
  end
21
30
 
22
31
  attr_reader :input_stream