iostreams 1.11.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +14 -13
- data/Rakefile +52 -0
- data/docs/CLAUDE.md +9 -0
- data/docs/config.md +157 -0
- data/docs/copy_files.md +75 -0
- data/docs/extensions.md +111 -0
- data/docs/formats.md +188 -0
- data/docs/index.md +388 -0
- data/docs/path.md +652 -0
- data/docs/pgp.md +436 -0
- data/docs/streams.md +337 -0
- data/docs/tutorial.md +483 -0
- data/docs/upgrading.md +217 -0
- data/lib/io_streams/builder.rb +71 -11
- data/lib/io_streams/bzip2/reader.rb +25 -2
- data/lib/io_streams/bzip2/writer.rb +26 -2
- data/lib/io_streams/encode/reader.rb +6 -2
- data/lib/io_streams/encode/writer.rb +9 -5
- data/lib/io_streams/errors.rb +4 -0
- data/lib/io_streams/gzip/reader.rb +5 -1
- data/lib/io_streams/gzip/writer.rb +11 -2
- data/lib/io_streams/io_streams.rb +156 -20
- data/lib/io_streams/line/reader.rb +9 -4
- data/lib/io_streams/line/writer.rb +1 -1
- data/lib/io_streams/path.rb +117 -8
- data/lib/io_streams/paths/file.rb +57 -11
- data/lib/io_streams/paths/http.rb +123 -9
- data/lib/io_streams/paths/matcher.rb +3 -3
- data/lib/io_streams/paths/s3.rb +69 -18
- data/lib/io_streams/paths/sftp/net_ssh.rb +104 -0
- data/lib/io_streams/paths/sftp.rb +103 -64
- data/lib/io_streams/pgp/reader.rb +63 -10
- data/lib/io_streams/pgp/writer.rb +111 -30
- data/lib/io_streams/pgp.rb +256 -71
- data/lib/io_streams/reader.rb +14 -5
- data/lib/io_streams/record/reader.rb +75 -6
- data/lib/io_streams/record/writer.rb +3 -4
- data/lib/io_streams/row/reader.rb +1 -1
- data/lib/io_streams/row/writer.rb +1 -1
- data/lib/io_streams/stream.rb +48 -37
- data/lib/io_streams/symmetric_encryption/reader.rb +6 -2
- data/lib/io_streams/symmetric_encryption/writer.rb +8 -4
- data/lib/io_streams/tabular/header.rb +49 -10
- data/lib/io_streams/tabular/parser/array.rb +0 -10
- data/lib/io_streams/tabular/parser/base.rb +10 -0
- data/lib/io_streams/tabular/parser/csv.rb +9 -36
- data/lib/io_streams/tabular/parser/fixed.rb +8 -6
- data/lib/io_streams/tabular/parser/psv.rb +6 -14
- data/lib/io_streams/tabular.rb +5 -10
- data/lib/io_streams/utils.rb +34 -2
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +16 -7
- data/lib/io_streams/xlsx/reader.rb +6 -2
- data/lib/io_streams/zip/reader.rb +4 -0
- data/lib/io_streams/zip/writer.rb +26 -10
- data/lib/iostreams.rb +0 -1
- metadata +46 -112
- data/lib/io_streams/deprecated.rb +0 -216
- data/lib/io_streams/tabular/utility/csv_row.rb +0 -105
- data/test/builder_test.rb +0 -311
- data/test/bzip2_reader_test.rb +0 -27
- data/test/bzip2_writer_test.rb +0 -56
- data/test/deprecated_test.rb +0 -121
- data/test/encode_reader_test.rb +0 -51
- data/test/encode_writer_test.rb +0 -90
- data/test/files/embedded_lines_test.csv +0 -7
- data/test/files/multiple_files.zip +0 -0
- data/test/files/spreadsheet.xlsx +0 -0
- data/test/files/test.csv +0 -4
- data/test/files/test.json +0 -3
- data/test/files/test.psv +0 -4
- data/test/files/text file.txt +0 -3
- data/test/files/text.txt +0 -3
- data/test/files/text.txt.bz2 +0 -0
- data/test/files/text.txt.gz +0 -0
- data/test/files/text.txt.gz.zip +0 -0
- data/test/files/text.zip +0 -0
- data/test/files/text.zip.gz +0 -0
- data/test/files/unclosed_quote_large_test.csv +0 -1658
- data/test/files/unclosed_quote_test.csv +0 -4
- data/test/files/unclosed_quote_test2.csv +0 -3
- data/test/gzip_reader_test.rb +0 -27
- data/test/gzip_writer_test.rb +0 -52
- data/test/io_streams_test.rb +0 -132
- data/test/line_reader_test.rb +0 -325
- data/test/line_writer_test.rb +0 -59
- data/test/minimal_file_reader.rb +0 -25
- data/test/path_test.rb +0 -55
- data/test/paths/file_test.rb +0 -213
- data/test/paths/http_test.rb +0 -34
- data/test/paths/matcher_test.rb +0 -120
- data/test/paths/s3_test.rb +0 -220
- data/test/paths/sftp_test.rb +0 -106
- data/test/pgp_reader_test.rb +0 -46
- data/test/pgp_test.rb +0 -267
- data/test/pgp_writer_test.rb +0 -130
- data/test/record_reader_test.rb +0 -60
- data/test/record_writer_test.rb +0 -82
- data/test/row_reader_test.rb +0 -35
- data/test/row_writer_test.rb +0 -56
- data/test/stream_test.rb +0 -577
- data/test/tabular_test.rb +0 -338
- data/test/test_helper.rb +0 -40
- data/test/utils_test.rb +0 -20
- data/test/xlsx_reader_test.rb +0 -37
- data/test/zip_reader_test.rb +0 -53
- data/test/zip_writer_test.rb +0 -48
data/lib/io_streams/pgp.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "open3"
|
|
2
|
+
require "shellwords"
|
|
2
3
|
module IOStreams
|
|
3
4
|
# Read/Write PGP/GPG file or stream.
|
|
4
5
|
#
|
|
@@ -25,6 +26,18 @@ module IOStreams
|
|
|
25
26
|
|
|
26
27
|
@executable = "gpg"
|
|
27
28
|
|
|
29
|
+
# Returns [Array<String>] the argv used to invoke gpg, with the supplied
|
|
30
|
+
# arguments appended.
|
|
31
|
+
#
|
|
32
|
+
# All gpg invocations are run without a shell (the multi-argument form of
|
|
33
|
+
# `Open3`) so that values such as email addresses, key ids, passphrases and
|
|
34
|
+
# file names cannot be interpreted as shell commands. The configured
|
|
35
|
+
# `executable` is split with `Shellwords` so that it may still contain
|
|
36
|
+
# additional fixed arguments (for example "gpg --homedir /path").
|
|
37
|
+
def self.gpg_command(*args)
|
|
38
|
+
Shellwords.split(executable) + args.map(&:to_s)
|
|
39
|
+
end
|
|
40
|
+
|
|
28
41
|
# Generate a new ultimate trusted local public and private key.
|
|
29
42
|
#
|
|
30
43
|
# Returns [String] the key id for the generated key.
|
|
@@ -44,6 +57,20 @@ module IOStreams
|
|
|
44
57
|
# Highly Recommended.
|
|
45
58
|
# To generate a good passphrase:
|
|
46
59
|
# `SecureRandom.urlsafe_base64(128)`
|
|
60
|
+
# Pass `nil` to generate an unprotected (passphrase-less) key.
|
|
61
|
+
#
|
|
62
|
+
# key_curve / subkey_curve [String]
|
|
63
|
+
# Optional Elliptic Curve to use for the (sub)key, e.g. "ed25519".
|
|
64
|
+
# When supplied the corresponding key/subkey length is ignored.
|
|
65
|
+
# Requires GnuPG 2.1 or later.
|
|
66
|
+
#
|
|
67
|
+
# key_usage / subkey_usage [String]
|
|
68
|
+
# Optional comma separated list of (sub)key capabilities, e.g. "sign".
|
|
69
|
+
# Requires GnuPG 2.1 or later.
|
|
70
|
+
#
|
|
71
|
+
# creation_date [String]
|
|
72
|
+
# Optional creation date for the key, e.g. "20240101T000000".
|
|
73
|
+
# Requires GnuPG 2.1 or later.
|
|
47
74
|
#
|
|
48
75
|
# See `man gpg` for the remaining options
|
|
49
76
|
def self.generate_key(name:,
|
|
@@ -54,23 +81,70 @@ module IOStreams
|
|
|
54
81
|
key_length: 4096,
|
|
55
82
|
subkey_type: "RSA",
|
|
56
83
|
subkey_length: key_length,
|
|
84
|
+
key_curve: nil,
|
|
85
|
+
key_usage: nil,
|
|
86
|
+
subkey_curve: nil,
|
|
87
|
+
subkey_usage: nil,
|
|
88
|
+
creation_date: nil,
|
|
57
89
|
expire_date: nil)
|
|
58
90
|
version_check
|
|
59
|
-
|
|
91
|
+
|
|
92
|
+
# Reject newlines so that a value cannot inject additional directives into
|
|
93
|
+
# the gpg batch key-generation parameter file.
|
|
94
|
+
reject_newlines!(name: name, email: email, comment: comment, passphrase: passphrase,
|
|
95
|
+
key_type: key_type, subkey_type: subkey_type, expire_date: expire_date,
|
|
96
|
+
key_curve: key_curve, key_usage: key_usage,
|
|
97
|
+
subkey_curve: subkey_curve, subkey_usage: subkey_usage,
|
|
98
|
+
creation_date: creation_date)
|
|
99
|
+
|
|
100
|
+
# `%no-protection`, and the Elliptic Curve / usage / creation-date directives
|
|
101
|
+
# were all introduced in GnuPG 2.1. Keep older versions working by only
|
|
102
|
+
# emitting them when a 2.1+ binary is detected. `--batch --gen-key` accepts
|
|
103
|
+
# all of these on 2.1+, so there is no need for the newer `--full-gen-key`.
|
|
104
|
+
modern = pgp_version.to_f >= 2.1
|
|
105
|
+
|
|
106
|
+
unless modern
|
|
107
|
+
new_options = {
|
|
108
|
+
key_curve: key_curve,
|
|
109
|
+
key_usage: key_usage,
|
|
110
|
+
subkey_curve: subkey_curve,
|
|
111
|
+
subkey_usage: subkey_usage,
|
|
112
|
+
creation_date: creation_date
|
|
113
|
+
}.compact
|
|
114
|
+
unless new_options.empty?
|
|
115
|
+
raise(ArgumentError,
|
|
116
|
+
"IOStreams::Pgp.generate_key: #{new_options.keys.join(', ')} require GnuPG 2.1 or later " \
|
|
117
|
+
"(detected #{pgp_version})")
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
params = +""
|
|
122
|
+
# `%no-protection` is a control statement and must precede the key parameters.
|
|
123
|
+
# GnuPG 2.1+ requires this explicit opt-out to create an unprotected key;
|
|
124
|
+
# older versions create one simply by omitting the Passphrase directive.
|
|
125
|
+
params << "%no-protection\n" if !passphrase && modern
|
|
60
126
|
params << "Key-Type: #{key_type}\n" if key_type
|
|
61
|
-
|
|
127
|
+
# Key-Length and Key-Curve are mutually exclusive: curves imply their own length.
|
|
128
|
+
params << "Key-Length: #{key_length}\n" if key_length && !key_curve
|
|
129
|
+
params << "Key-Curve: #{key_curve}\n" if key_curve
|
|
130
|
+
params << "Key-Usage: #{key_usage}\n" if key_usage
|
|
62
131
|
params << "Subkey-Type: #{subkey_type}\n" if subkey_type
|
|
63
|
-
params << "Subkey-Length: #{subkey_length}\n" if subkey_length
|
|
132
|
+
params << "Subkey-Length: #{subkey_length}\n" if subkey_length && !subkey_curve
|
|
133
|
+
params << "Subkey-Curve: #{subkey_curve}\n" if subkey_curve
|
|
134
|
+
params << "Subkey-Usage: #{subkey_usage}\n" if subkey_usage
|
|
64
135
|
params << "Name-Real: #{name}\n" if name
|
|
65
136
|
params << "Name-Comment: #{comment}\n" if comment
|
|
66
137
|
params << "Name-Email: #{email}\n" if email
|
|
67
138
|
params << "Expire-Date: #{expire_date}\n" if expire_date
|
|
139
|
+
params << "Creation-Date: #{creation_date}\n" if creation_date
|
|
68
140
|
params << "Passphrase: #{passphrase}\n" if passphrase
|
|
69
141
|
params << "%commit"
|
|
70
|
-
command = "#{executable} --batch --gen-key --no-tty"
|
|
71
142
|
|
|
72
|
-
|
|
73
|
-
|
|
143
|
+
command = gpg_command("--batch", "--gen-key", "--no-tty")
|
|
144
|
+
|
|
145
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: params)
|
|
146
|
+
# Do not log `params`, it contains the passphrase.
|
|
147
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.generate_key: #{command.shelljoin}\n#{err}#{out}" }
|
|
74
148
|
|
|
75
149
|
raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
|
|
76
150
|
|
|
@@ -86,13 +160,14 @@ module IOStreams
|
|
|
86
160
|
end
|
|
87
161
|
end
|
|
88
162
|
|
|
89
|
-
# Delete all private and public keys for a particular email.
|
|
163
|
+
# Delete all private and public keys for a particular email or key id.
|
|
90
164
|
#
|
|
91
165
|
# Returns false if no key was found.
|
|
92
166
|
# Raises an exception if it fails to delete the key.
|
|
167
|
+
# Raises ArgumentError when neither :email nor :key_id is supplied.
|
|
93
168
|
#
|
|
94
|
-
# email: [String]
|
|
95
|
-
# key_id: [String]
|
|
169
|
+
# email: [String] Email address for the key.
|
|
170
|
+
# key_id: [String] Id for the key.
|
|
96
171
|
#
|
|
97
172
|
# public: [true|false]
|
|
98
173
|
# Whether to delete the public key
|
|
@@ -102,6 +177,8 @@ module IOStreams
|
|
|
102
177
|
# Whether to delete the private key
|
|
103
178
|
# Default: false
|
|
104
179
|
def self.delete_keys(email: nil, key_id: nil, public: true, private: false)
|
|
180
|
+
raise(ArgumentError, "Either :email, or :key_id must be supplied") if email.nil? && key_id.nil?
|
|
181
|
+
|
|
105
182
|
version_check
|
|
106
183
|
# Version 2.1+ uses delete_public_or_private_keys
|
|
107
184
|
# Version < 2.1 uses delete_public_or_private_keys_v1
|
|
@@ -127,14 +204,18 @@ module IOStreams
|
|
|
127
204
|
# date: [String]
|
|
128
205
|
# name: [String]
|
|
129
206
|
# email: [String]
|
|
207
|
+
# private: [true|false]
|
|
208
|
+
# trust: [String]
|
|
130
209
|
# Returns [] if no keys were found.
|
|
131
210
|
def self.list_keys(email: nil, key_id: nil, private: false)
|
|
132
211
|
version_check
|
|
133
|
-
|
|
134
|
-
|
|
212
|
+
args = [private ? "--list-secret-keys" : "--list-keys"]
|
|
213
|
+
# `--` stops gpg from treating the email or key id as an option.
|
|
214
|
+
args += ["--", (email || key_id).to_s] if email || key_id
|
|
215
|
+
command = gpg_command(*args)
|
|
135
216
|
|
|
136
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
137
|
-
logger&.debug { "IOStreams::Pgp.list_keys: #{command}\n#{err}#{out}" }
|
|
217
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
218
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.list_keys: #{command.shelljoin}\n#{err}#{out}" }
|
|
138
219
|
if status.success? && out.length.positive?
|
|
139
220
|
parse_list_output(out)
|
|
140
221
|
else
|
|
@@ -156,12 +237,14 @@ module IOStreams
|
|
|
156
237
|
# date: [String]
|
|
157
238
|
# name: [String]
|
|
158
239
|
# email: [String]
|
|
240
|
+
# private: [true|false]
|
|
241
|
+
# trust: [String]
|
|
159
242
|
def self.key_info(key:)
|
|
160
243
|
version_check
|
|
161
|
-
command = "
|
|
244
|
+
command = gpg_command("--batch", "--no-tty")
|
|
162
245
|
|
|
163
|
-
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
|
164
|
-
logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
|
|
246
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
|
|
247
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.key_info: #{command.shelljoin}\n#{err}#{out}" }
|
|
165
248
|
|
|
166
249
|
# Try parsing even if we get an error - some versions of GPG return non-zero status but still output key info
|
|
167
250
|
unless (status.success? || err.include?("key ID") || out.include?("pub")) && out.length.positive?
|
|
@@ -176,28 +259,43 @@ module IOStreams
|
|
|
176
259
|
parse_list_output(out)
|
|
177
260
|
end
|
|
178
261
|
|
|
179
|
-
# Returns [String] containing all the public keys for the supplied email address.
|
|
262
|
+
# Returns [String] containing all the public keys for the supplied email address or key id.
|
|
263
|
+
#
|
|
264
|
+
# Raises ArgumentError when neither :email nor :key_id is supplied.
|
|
265
|
+
# Raises Pgp::Failure when `email: nil` is supplied without a :key_id, as before :key_id was added.
|
|
266
|
+
# In v3.0 this will raise ArgumentError instead.
|
|
180
267
|
#
|
|
181
268
|
# email: [String] Email address for requested key.
|
|
182
269
|
#
|
|
270
|
+
# key_id: [String] Id for the requested key.
|
|
271
|
+
#
|
|
183
272
|
# ascii: [true|false]
|
|
184
273
|
# Whether to export as ASCII text instead of binary format
|
|
185
274
|
# Default: true
|
|
186
|
-
def self.export(email
|
|
275
|
+
def self.export(email: (email_omitted = true) && nil, key_id: nil, ascii: true, private: false, passphrase: nil)
|
|
276
|
+
if email.nil? && key_id.nil?
|
|
277
|
+
raise(ArgumentError, "Either :email, or :key_id must be supplied") if email_omitted
|
|
278
|
+
|
|
279
|
+
raise(Pgp::Failure, "GPG Failed reading key: no email address or key id was supplied")
|
|
280
|
+
end
|
|
281
|
+
|
|
187
282
|
version_check
|
|
188
283
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
284
|
+
args = []
|
|
285
|
+
args += ["--pinentry-mode", "loopback"] if pgp_version.to_f >= 2.1
|
|
286
|
+
args << "--no-symkey-cache" if pgp_version.to_f >= 2.4
|
|
287
|
+
args << "--armor" if ascii
|
|
288
|
+
args += ["--no-tty", "--batch"]
|
|
289
|
+
# Supply the passphrase on stdin so that it is not visible in the process list.
|
|
290
|
+
args += ["--passphrase-fd", "0"]
|
|
291
|
+
args << (private ? "--export-secret-keys" : "--export")
|
|
292
|
+
args += ["--", (email || key_id).to_s]
|
|
293
|
+
command = gpg_command(*args)
|
|
196
294
|
|
|
197
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
198
|
-
logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
|
|
295
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: passphrase.to_s)
|
|
296
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.export: #{command.shelljoin}\n#{err}" }
|
|
199
297
|
|
|
200
|
-
raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
|
|
298
|
+
raise(Pgp::Failure, "GPG Failed reading key: #{email || key_id}: #{err}") unless status.success? && out.length.positive?
|
|
201
299
|
|
|
202
300
|
out
|
|
203
301
|
end
|
|
@@ -219,10 +317,10 @@ module IOStreams
|
|
|
219
317
|
# * Invalidated keys must be removed manually.
|
|
220
318
|
def self.import(key:)
|
|
221
319
|
version_check
|
|
222
|
-
command = "
|
|
320
|
+
command = gpg_command("--batch", "--import")
|
|
223
321
|
|
|
224
|
-
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
|
225
|
-
logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
|
|
322
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
|
|
323
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.import: #{command.shelljoin}\n#{err}#{out}" }
|
|
226
324
|
|
|
227
325
|
# Handle both old and new versions of GPG
|
|
228
326
|
# For older versions, the output is in err, for newer ones it might be in out
|
|
@@ -316,23 +414,74 @@ module IOStreams
|
|
|
316
414
|
raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
|
|
317
415
|
end
|
|
318
416
|
|
|
319
|
-
#
|
|
417
|
+
# Imports the supplied key and then marks it as trusted at the supplied trust level.
|
|
418
|
+
#
|
|
419
|
+
# Returns [String] email for the supplied key, or its key id when no email is present.
|
|
420
|
+
#
|
|
421
|
+
# key: [String]
|
|
422
|
+
# The public (or private) key to import and trust.
|
|
423
|
+
#
|
|
424
|
+
# trust_level: [Integer]
|
|
425
|
+
# The owner-trust level to assign to the imported key, the same levels used by `set_trust`:
|
|
426
|
+
# 1 : Undefined (no opinion)
|
|
427
|
+
# 2 : Never (do not trust)
|
|
428
|
+
# 3 : Marginal
|
|
429
|
+
# 4 : Full
|
|
430
|
+
# 5 : Ultimate
|
|
431
|
+
# Default: 5 : Ultimate
|
|
432
|
+
#
|
|
433
|
+
# SECURITY WARNING:
|
|
434
|
+
# Only import and trust keys received from a verified, trusted source.
|
|
435
|
+
# The default trust level is `5` (Ultimate), which tells GPG to treat the imported key
|
|
436
|
+
# as if it were one of your own keys. An ultimately trusted key is implicitly valid and
|
|
437
|
+
# can in turn confer validity on other keys it has signed. Importing an attacker supplied
|
|
438
|
+
# key at this level allows that attacker to impersonate other recipients.
|
|
439
|
+
# When the key cannot be fully verified, supply a lower `trust_level`.
|
|
320
440
|
#
|
|
321
441
|
# Notes:
|
|
322
442
|
# - If the same email address has multiple keys then only the first is currently trusted.
|
|
323
|
-
def self.import_and_trust(key:)
|
|
443
|
+
def self.import_and_trust(key:, trust_level: 5)
|
|
444
|
+
info = import_and_trust_key_info(key: key, trust_level: trust_level)
|
|
445
|
+
info[:email] || info[:key_id]
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
# Imports and trusts the supplied key, see #import_and_trust.
|
|
449
|
+
#
|
|
450
|
+
# Returns [String] the recipient to encrypt to: the key's fingerprint when gpg supplies it,
|
|
451
|
+
# otherwise the email address or key id returned by #import_and_trust.
|
|
452
|
+
#
|
|
453
|
+
# Encrypting to the fingerprint ensures that the imported key is used, since gpg looks up an
|
|
454
|
+
# email address in the keyring, where another key may have the same email address.
|
|
455
|
+
#
|
|
456
|
+
# Used internally by the PGP writer.
|
|
457
|
+
def self.import_and_trust_recipient(key:, trust_level: 5)
|
|
458
|
+
info = import_and_trust_key_info(key: key, trust_level: trust_level)
|
|
459
|
+
key_id = info[:key_id].to_s
|
|
460
|
+
# gpg v2.1 and later supply the full fingerprint: 40 hex digits for v4 keys, 64 for v5 and v6 keys.
|
|
461
|
+
# Earlier versions only supply a short key id, which is not unique.
|
|
462
|
+
return key_id if key_id.match?(/\A(\h{40}|\h{64})\z/)
|
|
463
|
+
|
|
464
|
+
info[:email] || info[:key_id]
|
|
465
|
+
end
|
|
466
|
+
|
|
467
|
+
# Returns [Hash] the key info for the supplied key, after importing and trusting it.
|
|
468
|
+
def self.import_and_trust_key_info(key:, trust_level:)
|
|
324
469
|
raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
|
|
325
470
|
|
|
326
|
-
|
|
471
|
+
infos = key_info(key: key)
|
|
472
|
+
info = infos.last&.dup || {}
|
|
473
|
+
# When a key has several user ids, only the entry for its first user id includes the key id.
|
|
474
|
+
info[:key_id] ||= infos.reverse_each.find { |entry| entry[:key_id] }&.fetch(:key_id)
|
|
327
475
|
|
|
328
|
-
email
|
|
329
|
-
key_id =
|
|
476
|
+
email = info[:email]
|
|
477
|
+
key_id = info[:key_id]
|
|
330
478
|
raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
|
|
331
479
|
|
|
332
480
|
import(key: key)
|
|
333
|
-
set_trust(email: email, key_id: key_id)
|
|
334
|
-
|
|
481
|
+
set_trust(email: email, key_id: key_id, level: trust_level)
|
|
482
|
+
info
|
|
335
483
|
end
|
|
484
|
+
private_class_method :import_and_trust_key_info
|
|
336
485
|
|
|
337
486
|
# Set the trust level for an existing key.
|
|
338
487
|
#
|
|
@@ -340,50 +489,75 @@ module IOStreams
|
|
|
340
489
|
# Returns nil if the email was not found
|
|
341
490
|
#
|
|
342
491
|
# After importing keys, they are not trusted and the relevant trust level must be set.
|
|
492
|
+
#
|
|
493
|
+
# key_id: [String]
|
|
494
|
+
# The fingerprint of the key, as hexadecimal digits only.
|
|
495
|
+
# Raises ArgumentError when it contains any other characters.
|
|
496
|
+
#
|
|
497
|
+
# level: [Integer]
|
|
498
|
+
# The owner-trust level to assign to the key:
|
|
499
|
+
# 1 : Undefined (no opinion)
|
|
500
|
+
# 2 : Never (do not trust)
|
|
501
|
+
# 3 : Marginal
|
|
502
|
+
# 4 : Full
|
|
503
|
+
# 5 : Ultimate
|
|
343
504
|
# Default: 5 : Ultimate
|
|
505
|
+
#
|
|
506
|
+
# SECURITY WARNING:
|
|
507
|
+
# Only trust keys received from a verified, trusted source.
|
|
508
|
+
# The default trust level is `5` (Ultimate), which tells GPG to treat the key
|
|
509
|
+
# as if it were one of your own keys. An ultimately trusted key is implicitly valid and
|
|
510
|
+
# can in turn confer validity on other keys it has signed. Trusting an attacker supplied
|
|
511
|
+
# key at this level allows that attacker to impersonate other recipients.
|
|
512
|
+
# When the key cannot be fully verified, supply a lower `level`.
|
|
344
513
|
def self.set_trust(email: nil, key_id: nil, level: 5)
|
|
514
|
+
# The key_id is written into gpg's ownertrust input, where any other character, such as a newline,
|
|
515
|
+
# could add trust lines for other keys.
|
|
516
|
+
if key_id && !key_id.to_s.match?(/\A\h+\z/)
|
|
517
|
+
raise(ArgumentError, "Invalid :key_id, it must only contain hexadecimal digits: #{key_id.inspect}")
|
|
518
|
+
end
|
|
519
|
+
|
|
345
520
|
version_check
|
|
346
521
|
fingerprint = key_id || fingerprint(email: email)
|
|
347
522
|
return unless fingerprint
|
|
348
523
|
|
|
349
|
-
command = "
|
|
524
|
+
command = gpg_command("--import-ownertrust")
|
|
350
525
|
trust = "#{fingerprint}:#{level + 1}:\n"
|
|
351
|
-
out, err, status = Open3.capture3(command, stdin_data: trust)
|
|
352
|
-
logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
|
|
526
|
+
out, err, status = Open3.capture3(*command, stdin_data: trust)
|
|
527
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.set_trust: #{command.shelljoin}\n#{err}#{out}" }
|
|
353
528
|
|
|
354
529
|
raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
|
|
355
530
|
|
|
356
531
|
err
|
|
357
532
|
end
|
|
358
533
|
|
|
359
|
-
#
|
|
534
|
+
# Internal: resolve an email address to a key fingerprint.
|
|
535
|
+
# Public callers should identify keys by `key_id` (see #list_keys / #key_info).
|
|
360
536
|
def self.fingerprint(email:)
|
|
361
537
|
version_check
|
|
362
|
-
|
|
538
|
+
command = gpg_command("--list-keys", "--fingerprint", "--with-colons", "--", email.to_s)
|
|
539
|
+
Open3.popen2e(*command) do |_stdin, out, waith_thr|
|
|
363
540
|
output = out.read.chomp
|
|
364
|
-
if !waith_thr.value.success? &&
|
|
541
|
+
if !waith_thr.value.success? && output !~ /(public key not found|No public key)/i
|
|
365
542
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
|
|
366
543
|
end
|
|
367
544
|
|
|
368
545
|
output.each_line do |line|
|
|
369
|
-
if (match = line.match(/\Afpr.*::([
|
|
546
|
+
if (match = line.match(/\Afpr.*::([^:]*):\Z/))
|
|
370
547
|
return match[1]
|
|
371
548
|
end
|
|
372
549
|
end
|
|
373
550
|
nil
|
|
374
551
|
end
|
|
375
552
|
end
|
|
376
|
-
|
|
377
|
-
def self.logger=(logger)
|
|
378
|
-
@logger = logger
|
|
379
|
-
end
|
|
553
|
+
private_class_method :fingerprint
|
|
380
554
|
|
|
381
555
|
# Returns [String] the version of pgp currently installed
|
|
382
556
|
def self.pgp_version
|
|
383
557
|
@pgp_version ||= begin
|
|
384
|
-
command = "
|
|
385
|
-
out, err, status = Open3.capture3(command)
|
|
386
|
-
logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
|
|
558
|
+
command = gpg_command("--version")
|
|
559
|
+
out, err, status = Open3.capture3(*command)
|
|
560
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.version: #{command.shelljoin}\n#{err}#{out}" }
|
|
387
561
|
if status.success?
|
|
388
562
|
# Sample output
|
|
389
563
|
# #{executable} (GnuPG) 2.0.30
|
|
@@ -413,12 +587,6 @@ module IOStreams
|
|
|
413
587
|
end
|
|
414
588
|
end
|
|
415
589
|
|
|
416
|
-
@logger = nil
|
|
417
|
-
|
|
418
|
-
def self.logger
|
|
419
|
-
@logger
|
|
420
|
-
end
|
|
421
|
-
|
|
422
590
|
def self.version_check
|
|
423
591
|
# Previously, this method raised an error for versions >= 2.4
|
|
424
592
|
# Now we support versions up to and including 2.4.7
|
|
@@ -497,16 +665,24 @@ module IOStreams
|
|
|
497
665
|
hash[:trust] = match[2].to_s.strip if match[1]
|
|
498
666
|
results << hash
|
|
499
667
|
hash = {}
|
|
500
|
-
elsif (match = line.match(/\s+([A-Z0-9]{16,
|
|
668
|
+
elsif (match = line.match(/\s+([A-Z0-9]{16,64})/))
|
|
501
669
|
# v2.2/v2.4 key id on separate line:
|
|
502
670
|
# 18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
|
|
503
671
|
# Or shorter format: 7932AB23D7238F6B
|
|
672
|
+
# Or a 64 digit fingerprint for v5 and v6 keys.
|
|
504
673
|
hash[:key_id] ||= match[1]
|
|
505
674
|
end
|
|
506
675
|
end
|
|
507
676
|
results
|
|
508
677
|
end
|
|
509
678
|
|
|
679
|
+
def self.reject_newlines!(**fields)
|
|
680
|
+
fields.each_pair do |field, value|
|
|
681
|
+
next if value.nil?
|
|
682
|
+
raise(ArgumentError, "IOStreams::Pgp.generate_key: :#{field} cannot contain newlines") if value.to_s =~ /[\r\n]/
|
|
683
|
+
end
|
|
684
|
+
end
|
|
685
|
+
|
|
510
686
|
def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
|
|
511
687
|
keys = private ? "secret-keys" : "keys"
|
|
512
688
|
|
|
@@ -517,9 +693,9 @@ module IOStreams
|
|
|
517
693
|
key_id = key_info[:key_id]
|
|
518
694
|
next unless key_id
|
|
519
695
|
|
|
520
|
-
command = "
|
|
521
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
522
|
-
logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
|
|
696
|
+
command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", key_id)
|
|
697
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
698
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}#{out}" }
|
|
523
699
|
|
|
524
700
|
unless status.success?
|
|
525
701
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
|
@@ -532,18 +708,27 @@ module IOStreams
|
|
|
532
708
|
def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
|
|
533
709
|
keys = private ? "secret-keys" : "keys"
|
|
534
710
|
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
711
|
+
# List the fingerprints, then delete each one. Previously this shelled out
|
|
712
|
+
# to a `for` loop, which allowed shell injection via :email / :key_id.
|
|
713
|
+
list_command = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", "--", (email || key_id).to_s)
|
|
714
|
+
list_out, list_err, = Open3.capture3(*list_command, binmode: true)
|
|
715
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{list_command.shelljoin}\n#{list_err}: #{list_out}" }
|
|
716
|
+
|
|
717
|
+
return false if list_err =~ /(not found|no public key)/i
|
|
718
|
+
|
|
719
|
+
fingerprints = list_out.each_line.select { |line| line.start_with?("fpr") }.map { |line| line.split(":")[9] }.compact
|
|
720
|
+
return false if fingerprints.empty?
|
|
538
721
|
|
|
539
|
-
|
|
540
|
-
|
|
722
|
+
fingerprints.each do |fingerprint|
|
|
723
|
+
command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", "--", fingerprint)
|
|
724
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
725
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}: #{out}" }
|
|
541
726
|
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
727
|
+
unless status.success?
|
|
728
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
|
729
|
+
end
|
|
730
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
|
|
545
731
|
end
|
|
546
|
-
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
|
|
547
732
|
|
|
548
733
|
true
|
|
549
734
|
end
|
data/lib/io_streams/reader.rb
CHANGED
|
@@ -1,22 +1,31 @@
|
|
|
1
1
|
module IOStreams
|
|
2
2
|
class Reader
|
|
3
|
+
# Returns [Array<Symbol>] the names of the options this reader accepts,
|
|
4
|
+
# or [nil] when the reader does not declare them.
|
|
5
|
+
#
|
|
6
|
+
# When declared, `IOStreams::Builder` rejects any other option before the reader is opened,
|
|
7
|
+
# naming the direction an option belongs to when it is only valid for the other direction.
|
|
8
|
+
def self.option_names
|
|
9
|
+
nil
|
|
10
|
+
end
|
|
11
|
+
|
|
3
12
|
# When a Reader does not support streams, we copy the stream to a local temp file
|
|
4
13
|
# and then pass that filename in for this reader.
|
|
5
14
|
def self.stream(input_stream, **args, &block)
|
|
6
|
-
Utils.
|
|
15
|
+
Utils.private_temp_file("iostreams_reader") do |file_name|
|
|
7
16
|
::File.open(file_name, "wb") { |target| ::IO.copy_stream(input_stream, target) }
|
|
8
17
|
file(file_name, **args, &block)
|
|
9
18
|
end
|
|
10
19
|
end
|
|
11
20
|
|
|
12
21
|
# When a Writer supports streams, also allow it to simply support a file
|
|
13
|
-
def self.file(file_name,
|
|
14
|
-
::File.open(file_name, "rb") { |file| stream(file,
|
|
22
|
+
def self.file(file_name, **args, &block)
|
|
23
|
+
::File.open(file_name, "rb") { |file| stream(file, **args, &block) }
|
|
15
24
|
end
|
|
16
25
|
|
|
17
26
|
# For processing by either a file name or an open IO stream.
|
|
18
|
-
def self.open(file_name_or_io, **args, &
|
|
19
|
-
file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &
|
|
27
|
+
def self.open(file_name_or_io, **args, &)
|
|
28
|
+
file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
|
|
20
29
|
end
|
|
21
30
|
|
|
22
31
|
attr_reader :input_stream
|