iostreams 1.11.0 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +20 -2
- data/Rakefile +7 -0
- data/lib/io_streams/builder.rb +9 -9
- data/lib/io_streams/bzip2/writer.rb +1 -1
- data/lib/io_streams/encode/reader.rb +2 -2
- data/lib/io_streams/encode/writer.rb +5 -5
- data/lib/io_streams/gzip/reader.rb +1 -1
- data/lib/io_streams/gzip/writer.rb +1 -1
- data/lib/io_streams/io_streams.rb +45 -19
- data/lib/io_streams/line/reader.rb +2 -2
- data/lib/io_streams/line/writer.rb +1 -1
- data/lib/io_streams/path.rb +2 -2
- data/lib/io_streams/paths/file.rb +10 -10
- data/lib/io_streams/paths/http.rb +80 -7
- data/lib/io_streams/paths/matcher.rb +3 -3
- data/lib/io_streams/paths/s3.rb +3 -3
- data/lib/io_streams/paths/sftp.rb +7 -8
- data/lib/io_streams/pgp/reader.rb +23 -10
- data/lib/io_streams/pgp/writer.rb +93 -32
- data/lib/io_streams/pgp.rb +188 -60
- data/lib/io_streams/reader.rb +4 -4
- data/lib/io_streams/record/reader.rb +3 -4
- data/lib/io_streams/record/writer.rb +3 -4
- data/lib/io_streams/row/reader.rb +1 -1
- data/lib/io_streams/row/writer.rb +1 -1
- data/lib/io_streams/stream.rb +36 -30
- data/lib/io_streams/symmetric_encryption/reader.rb +2 -2
- data/lib/io_streams/symmetric_encryption/writer.rb +4 -4
- data/lib/io_streams/tabular/header.rb +18 -6
- data/lib/io_streams/tabular/parser/array.rb +0 -10
- data/lib/io_streams/tabular/parser/csv.rb +6 -38
- data/lib/io_streams/tabular/parser/fixed.rb +5 -5
- data/lib/io_streams/tabular/parser/psv.rb +0 -12
- data/lib/io_streams/tabular.rb +5 -10
- data/lib/io_streams/utils.rb +3 -2
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +6 -6
- data/lib/io_streams/xlsx/reader.rb +1 -1
- data/lib/io_streams/zip/writer.rb +22 -10
- data/lib/iostreams.rb +0 -1
- metadata +28 -111
- data/lib/io_streams/deprecated.rb +0 -216
- data/lib/io_streams/tabular/utility/csv_row.rb +0 -105
- data/test/builder_test.rb +0 -311
- data/test/bzip2_reader_test.rb +0 -27
- data/test/bzip2_writer_test.rb +0 -56
- data/test/deprecated_test.rb +0 -121
- data/test/encode_reader_test.rb +0 -51
- data/test/encode_writer_test.rb +0 -90
- data/test/files/embedded_lines_test.csv +0 -7
- data/test/files/multiple_files.zip +0 -0
- data/test/files/spreadsheet.xlsx +0 -0
- data/test/files/test.csv +0 -4
- data/test/files/test.json +0 -3
- data/test/files/test.psv +0 -4
- data/test/files/text file.txt +0 -3
- data/test/files/text.txt +0 -3
- data/test/files/text.txt.bz2 +0 -0
- data/test/files/text.txt.gz +0 -0
- data/test/files/text.txt.gz.zip +0 -0
- data/test/files/text.zip +0 -0
- data/test/files/text.zip.gz +0 -0
- data/test/files/unclosed_quote_large_test.csv +0 -1658
- data/test/files/unclosed_quote_test.csv +0 -4
- data/test/files/unclosed_quote_test2.csv +0 -3
- data/test/gzip_reader_test.rb +0 -27
- data/test/gzip_writer_test.rb +0 -52
- data/test/io_streams_test.rb +0 -132
- data/test/line_reader_test.rb +0 -325
- data/test/line_writer_test.rb +0 -59
- data/test/minimal_file_reader.rb +0 -25
- data/test/path_test.rb +0 -55
- data/test/paths/file_test.rb +0 -213
- data/test/paths/http_test.rb +0 -34
- data/test/paths/matcher_test.rb +0 -120
- data/test/paths/s3_test.rb +0 -220
- data/test/paths/sftp_test.rb +0 -106
- data/test/pgp_reader_test.rb +0 -46
- data/test/pgp_test.rb +0 -267
- data/test/pgp_writer_test.rb +0 -130
- data/test/record_reader_test.rb +0 -60
- data/test/record_writer_test.rb +0 -82
- data/test/row_reader_test.rb +0 -35
- data/test/row_writer_test.rb +0 -56
- data/test/stream_test.rb +0 -577
- data/test/tabular_test.rb +0 -338
- data/test/test_helper.rb +0 -40
- data/test/utils_test.rb +0 -20
- data/test/xlsx_reader_test.rb +0 -37
- data/test/zip_reader_test.rb +0 -53
- data/test/zip_writer_test.rb +0 -48
data/lib/io_streams/pgp.rb
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
require "open3"
|
|
2
|
+
require "shellwords"
|
|
2
3
|
module IOStreams
|
|
3
4
|
# Read/Write PGP/GPG file or stream.
|
|
4
5
|
#
|
|
@@ -25,6 +26,18 @@ module IOStreams
|
|
|
25
26
|
|
|
26
27
|
@executable = "gpg"
|
|
27
28
|
|
|
29
|
+
# Returns [Array<String>] the argv used to invoke gpg, with the supplied
|
|
30
|
+
# arguments appended.
|
|
31
|
+
#
|
|
32
|
+
# All gpg invocations are run without a shell (the multi-argument form of
|
|
33
|
+
# `Open3`) so that values such as email addresses, key ids, passphrases and
|
|
34
|
+
# file names cannot be interpreted as shell commands. The configured
|
|
35
|
+
# `executable` is split with `Shellwords` so that it may still contain
|
|
36
|
+
# additional fixed arguments (for example "gpg --homedir /path").
|
|
37
|
+
def self.gpg_command(*args)
|
|
38
|
+
Shellwords.split(executable) + args.map(&:to_s)
|
|
39
|
+
end
|
|
40
|
+
|
|
28
41
|
# Generate a new ultimate trusted local public and private key.
|
|
29
42
|
#
|
|
30
43
|
# Returns [String] the key id for the generated key.
|
|
@@ -44,6 +57,20 @@ module IOStreams
|
|
|
44
57
|
# Highly Recommended.
|
|
45
58
|
# To generate a good passphrase:
|
|
46
59
|
# `SecureRandom.urlsafe_base64(128)`
|
|
60
|
+
# Pass `nil` to generate an unprotected (passphrase-less) key.
|
|
61
|
+
#
|
|
62
|
+
# key_curve / subkey_curve [String]
|
|
63
|
+
# Optional Elliptic Curve to use for the (sub)key, e.g. "ed25519".
|
|
64
|
+
# When supplied the corresponding key/subkey length is ignored.
|
|
65
|
+
# Requires GnuPG 2.1 or later.
|
|
66
|
+
#
|
|
67
|
+
# key_usage / subkey_usage [String]
|
|
68
|
+
# Optional comma separated list of (sub)key capabilities, e.g. "sign".
|
|
69
|
+
# Requires GnuPG 2.1 or later.
|
|
70
|
+
#
|
|
71
|
+
# creation_date [String]
|
|
72
|
+
# Optional creation date for the key, e.g. "20240101T000000".
|
|
73
|
+
# Requires GnuPG 2.1 or later.
|
|
47
74
|
#
|
|
48
75
|
# See `man gpg` for the remaining options
|
|
49
76
|
def self.generate_key(name:,
|
|
@@ -54,23 +81,70 @@ module IOStreams
|
|
|
54
81
|
key_length: 4096,
|
|
55
82
|
subkey_type: "RSA",
|
|
56
83
|
subkey_length: key_length,
|
|
84
|
+
key_curve: nil,
|
|
85
|
+
key_usage: nil,
|
|
86
|
+
subkey_curve: nil,
|
|
87
|
+
subkey_usage: nil,
|
|
88
|
+
creation_date: nil,
|
|
57
89
|
expire_date: nil)
|
|
58
90
|
version_check
|
|
59
|
-
|
|
91
|
+
|
|
92
|
+
# Reject newlines so that a value cannot inject additional directives into
|
|
93
|
+
# the gpg batch key-generation parameter file.
|
|
94
|
+
reject_newlines!(name: name, email: email, comment: comment, passphrase: passphrase,
|
|
95
|
+
key_type: key_type, subkey_type: subkey_type, expire_date: expire_date,
|
|
96
|
+
key_curve: key_curve, key_usage: key_usage,
|
|
97
|
+
subkey_curve: subkey_curve, subkey_usage: subkey_usage,
|
|
98
|
+
creation_date: creation_date)
|
|
99
|
+
|
|
100
|
+
# `%no-protection`, and the Elliptic Curve / usage / creation-date directives
|
|
101
|
+
# were all introduced in GnuPG 2.1. Keep older versions working by only
|
|
102
|
+
# emitting them when a 2.1+ binary is detected. `--batch --gen-key` accepts
|
|
103
|
+
# all of these on 2.1+, so there is no need for the newer `--full-gen-key`.
|
|
104
|
+
modern = pgp_version.to_f >= 2.1
|
|
105
|
+
|
|
106
|
+
unless modern
|
|
107
|
+
new_options = {
|
|
108
|
+
key_curve: key_curve,
|
|
109
|
+
key_usage: key_usage,
|
|
110
|
+
subkey_curve: subkey_curve,
|
|
111
|
+
subkey_usage: subkey_usage,
|
|
112
|
+
creation_date: creation_date
|
|
113
|
+
}.compact
|
|
114
|
+
unless new_options.empty?
|
|
115
|
+
raise(ArgumentError,
|
|
116
|
+
"IOStreams::Pgp.generate_key: #{new_options.keys.join(', ')} require GnuPG 2.1 or later " \
|
|
117
|
+
"(detected #{pgp_version})")
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
params = +""
|
|
122
|
+
# `%no-protection` is a control statement and must precede the key parameters.
|
|
123
|
+
# GnuPG 2.1+ requires this explicit opt-out to create an unprotected key;
|
|
124
|
+
# older versions create one simply by omitting the Passphrase directive.
|
|
125
|
+
params << "%no-protection\n" if !passphrase && modern
|
|
60
126
|
params << "Key-Type: #{key_type}\n" if key_type
|
|
61
|
-
|
|
127
|
+
# Key-Length and Key-Curve are mutually exclusive: curves imply their own length.
|
|
128
|
+
params << "Key-Length: #{key_length}\n" if key_length && !key_curve
|
|
129
|
+
params << "Key-Curve: #{key_curve}\n" if key_curve
|
|
130
|
+
params << "Key-Usage: #{key_usage}\n" if key_usage
|
|
62
131
|
params << "Subkey-Type: #{subkey_type}\n" if subkey_type
|
|
63
|
-
params << "Subkey-Length: #{subkey_length}\n" if subkey_length
|
|
132
|
+
params << "Subkey-Length: #{subkey_length}\n" if subkey_length && !subkey_curve
|
|
133
|
+
params << "Subkey-Curve: #{subkey_curve}\n" if subkey_curve
|
|
134
|
+
params << "Subkey-Usage: #{subkey_usage}\n" if subkey_usage
|
|
64
135
|
params << "Name-Real: #{name}\n" if name
|
|
65
136
|
params << "Name-Comment: #{comment}\n" if comment
|
|
66
137
|
params << "Name-Email: #{email}\n" if email
|
|
67
138
|
params << "Expire-Date: #{expire_date}\n" if expire_date
|
|
139
|
+
params << "Creation-Date: #{creation_date}\n" if creation_date
|
|
68
140
|
params << "Passphrase: #{passphrase}\n" if passphrase
|
|
69
141
|
params << "%commit"
|
|
70
|
-
command = "#{executable} --batch --gen-key --no-tty"
|
|
71
142
|
|
|
72
|
-
|
|
73
|
-
|
|
143
|
+
command = gpg_command("--batch", "--gen-key", "--no-tty")
|
|
144
|
+
|
|
145
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: params)
|
|
146
|
+
# Do not log `params`, it contains the passphrase.
|
|
147
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.generate_key: #{command.shelljoin}\n#{err}#{out}" }
|
|
74
148
|
|
|
75
149
|
raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
|
|
76
150
|
|
|
@@ -127,14 +201,17 @@ module IOStreams
|
|
|
127
201
|
# date: [String]
|
|
128
202
|
# name: [String]
|
|
129
203
|
# email: [String]
|
|
204
|
+
# private: [true|false]
|
|
205
|
+
# trust: [String]
|
|
130
206
|
# Returns [] if no keys were found.
|
|
131
207
|
def self.list_keys(email: nil, key_id: nil, private: false)
|
|
132
208
|
version_check
|
|
133
|
-
|
|
134
|
-
|
|
209
|
+
args = [private ? "--list-secret-keys" : "--list-keys"]
|
|
210
|
+
args << (email || key_id).to_s if email || key_id
|
|
211
|
+
command = gpg_command(*args)
|
|
135
212
|
|
|
136
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
137
|
-
logger&.debug { "IOStreams::Pgp.list_keys: #{command}\n#{err}#{out}" }
|
|
213
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
214
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.list_keys: #{command.shelljoin}\n#{err}#{out}" }
|
|
138
215
|
if status.success? && out.length.positive?
|
|
139
216
|
parse_list_output(out)
|
|
140
217
|
else
|
|
@@ -156,12 +233,14 @@ module IOStreams
|
|
|
156
233
|
# date: [String]
|
|
157
234
|
# name: [String]
|
|
158
235
|
# email: [String]
|
|
236
|
+
# private: [true|false]
|
|
237
|
+
# trust: [String]
|
|
159
238
|
def self.key_info(key:)
|
|
160
239
|
version_check
|
|
161
|
-
command = "
|
|
240
|
+
command = gpg_command("--batch", "--no-tty")
|
|
162
241
|
|
|
163
|
-
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
|
164
|
-
logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
|
|
242
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
|
|
243
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.key_info: #{command.shelljoin}\n#{err}#{out}" }
|
|
165
244
|
|
|
166
245
|
# Try parsing even if we get an error - some versions of GPG return non-zero status but still output key info
|
|
167
246
|
unless (status.success? || err.include?("key ID") || out.include?("pub")) && out.length.positive?
|
|
@@ -186,16 +265,18 @@ module IOStreams
|
|
|
186
265
|
def self.export(email:, ascii: true, private: false, passphrase: nil)
|
|
187
266
|
version_check
|
|
188
267
|
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
268
|
+
args = []
|
|
269
|
+
args += ["--pinentry-mode", "loopback"] if pgp_version.to_f >= 2.1
|
|
270
|
+
args << "--no-symkey-cache" if pgp_version.to_f >= 2.4
|
|
271
|
+
args << "--armor" if ascii
|
|
272
|
+
args += ["--no-tty", "--batch"]
|
|
273
|
+
args += passphrase ? ["--passphrase", passphrase] : ["--passphrase-fd", "0"]
|
|
274
|
+
args += private ? ["--export-secret-keys", email.to_s] : ["--export", email.to_s]
|
|
275
|
+
command = gpg_command(*args)
|
|
196
276
|
|
|
197
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
198
|
-
|
|
277
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
278
|
+
# Do not log the command, it may contain the passphrase.
|
|
279
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.export: #{email}\n#{err}" }
|
|
199
280
|
|
|
200
281
|
raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
|
|
201
282
|
|
|
@@ -219,10 +300,10 @@ module IOStreams
|
|
|
219
300
|
# * Invalidated keys must be removed manually.
|
|
220
301
|
def self.import(key:)
|
|
221
302
|
version_check
|
|
222
|
-
command = "
|
|
303
|
+
command = gpg_command("--batch", "--import")
|
|
223
304
|
|
|
224
|
-
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
|
225
|
-
logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
|
|
305
|
+
out, err, status = Open3.capture3(*command, binmode: true, stdin_data: key)
|
|
306
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.import: #{command.shelljoin}\n#{err}#{out}" }
|
|
226
307
|
|
|
227
308
|
# Handle both old and new versions of GPG
|
|
228
309
|
# For older versions, the output is in err, for newer ones it might be in out
|
|
@@ -316,11 +397,33 @@ module IOStreams
|
|
|
316
397
|
raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
|
|
317
398
|
end
|
|
318
399
|
|
|
319
|
-
#
|
|
400
|
+
# Imports the supplied key and then marks it as trusted at the supplied trust level.
|
|
401
|
+
#
|
|
402
|
+
# Returns [String] email for the supplied key, or its key id when no email is present.
|
|
403
|
+
#
|
|
404
|
+
# key: [String]
|
|
405
|
+
# The public (or private) key to import and trust.
|
|
406
|
+
#
|
|
407
|
+
# trust_level: [Integer]
|
|
408
|
+
# The owner-trust level to assign to the imported key, the same levels used by `set_trust`:
|
|
409
|
+
# 1 : Undefined (no opinion)
|
|
410
|
+
# 2 : Never (do not trust)
|
|
411
|
+
# 3 : Marginal
|
|
412
|
+
# 4 : Full
|
|
413
|
+
# 5 : Ultimate
|
|
414
|
+
# Default: 5 : Ultimate
|
|
415
|
+
#
|
|
416
|
+
# SECURITY WARNING:
|
|
417
|
+
# Only import and trust keys received from a verified, trusted source.
|
|
418
|
+
# The default trust level is `5` (Ultimate), which tells GPG to treat the imported key
|
|
419
|
+
# as if it were one of your own keys. An ultimately trusted key is implicitly valid and
|
|
420
|
+
# can in turn confer validity on other keys it has signed. Importing an attacker supplied
|
|
421
|
+
# key at this level allows that attacker to impersonate other recipients.
|
|
422
|
+
# When the key cannot be fully verified, supply a lower `trust_level`.
|
|
320
423
|
#
|
|
321
424
|
# Notes:
|
|
322
425
|
# - If the same email address has multiple keys then only the first is currently trusted.
|
|
323
|
-
def self.import_and_trust(key:)
|
|
426
|
+
def self.import_and_trust(key:, trust_level: 5)
|
|
324
427
|
raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
|
|
325
428
|
|
|
326
429
|
key_info = key_info(key: key).last
|
|
@@ -330,7 +433,7 @@ module IOStreams
|
|
|
330
433
|
raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
|
|
331
434
|
|
|
332
435
|
import(key: key)
|
|
333
|
-
set_trust(email: email, key_id: key_id)
|
|
436
|
+
set_trust(email: email, key_id: key_id, level: trust_level)
|
|
334
437
|
email || key_id
|
|
335
438
|
end
|
|
336
439
|
|
|
@@ -340,50 +443,65 @@ module IOStreams
|
|
|
340
443
|
# Returns nil if the email was not found
|
|
341
444
|
#
|
|
342
445
|
# After importing keys, they are not trusted and the relevant trust level must be set.
|
|
446
|
+
#
|
|
447
|
+
# level: [Integer]
|
|
448
|
+
# The owner-trust level to assign to the key:
|
|
449
|
+
# 1 : Undefined (no opinion)
|
|
450
|
+
# 2 : Never (do not trust)
|
|
451
|
+
# 3 : Marginal
|
|
452
|
+
# 4 : Full
|
|
453
|
+
# 5 : Ultimate
|
|
343
454
|
# Default: 5 : Ultimate
|
|
455
|
+
#
|
|
456
|
+
# SECURITY WARNING:
|
|
457
|
+
# Only trust keys received from a verified, trusted source.
|
|
458
|
+
# The default trust level is `5` (Ultimate), which tells GPG to treat the key
|
|
459
|
+
# as if it were one of your own keys. An ultimately trusted key is implicitly valid and
|
|
460
|
+
# can in turn confer validity on other keys it has signed. Trusting an attacker supplied
|
|
461
|
+
# key at this level allows that attacker to impersonate other recipients.
|
|
462
|
+
# When the key cannot be fully verified, supply a lower `level`.
|
|
344
463
|
def self.set_trust(email: nil, key_id: nil, level: 5)
|
|
345
464
|
version_check
|
|
346
465
|
fingerprint = key_id || fingerprint(email: email)
|
|
347
466
|
return unless fingerprint
|
|
348
467
|
|
|
349
|
-
command = "
|
|
468
|
+
command = gpg_command("--import-ownertrust")
|
|
350
469
|
trust = "#{fingerprint}:#{level + 1}:\n"
|
|
351
|
-
out, err, status = Open3.capture3(command, stdin_data: trust)
|
|
352
|
-
logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
|
|
470
|
+
out, err, status = Open3.capture3(*command, stdin_data: trust)
|
|
471
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.set_trust: #{command.shelljoin}\n#{err}#{out}" }
|
|
353
472
|
|
|
354
473
|
raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
|
|
355
474
|
|
|
356
475
|
err
|
|
357
476
|
end
|
|
358
477
|
|
|
359
|
-
#
|
|
478
|
+
# Internal: resolve an email address to a key fingerprint.
|
|
479
|
+
# Public callers should identify keys by `key_id` (see #list_keys / #key_info).
|
|
360
480
|
def self.fingerprint(email:)
|
|
361
481
|
version_check
|
|
362
|
-
|
|
482
|
+
command = gpg_command("--list-keys", "--fingerprint", "--with-colons", email.to_s)
|
|
483
|
+
Open3.popen2e(*command) do |_stdin, out, waith_thr|
|
|
363
484
|
output = out.read.chomp
|
|
364
|
-
if !waith_thr.value.success? &&
|
|
485
|
+
if !waith_thr.value.success? && output !~ /(public key not found|No public key)/i
|
|
365
486
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
|
|
366
487
|
end
|
|
367
488
|
|
|
368
489
|
output.each_line do |line|
|
|
369
|
-
if (match = line.match(/\Afpr.*::([
|
|
490
|
+
if (match = line.match(/\Afpr.*::([^:]*):\Z/))
|
|
370
491
|
return match[1]
|
|
371
492
|
end
|
|
372
493
|
end
|
|
373
494
|
nil
|
|
374
495
|
end
|
|
375
496
|
end
|
|
376
|
-
|
|
377
|
-
def self.logger=(logger)
|
|
378
|
-
@logger = logger
|
|
379
|
-
end
|
|
497
|
+
private_class_method :fingerprint
|
|
380
498
|
|
|
381
499
|
# Returns [String] the version of pgp currently installed
|
|
382
500
|
def self.pgp_version
|
|
383
501
|
@pgp_version ||= begin
|
|
384
|
-
command = "
|
|
385
|
-
out, err, status = Open3.capture3(command)
|
|
386
|
-
logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
|
|
502
|
+
command = gpg_command("--version")
|
|
503
|
+
out, err, status = Open3.capture3(*command)
|
|
504
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.version: #{command.shelljoin}\n#{err}#{out}" }
|
|
387
505
|
if status.success?
|
|
388
506
|
# Sample output
|
|
389
507
|
# #{executable} (GnuPG) 2.0.30
|
|
@@ -413,12 +531,6 @@ module IOStreams
|
|
|
413
531
|
end
|
|
414
532
|
end
|
|
415
533
|
|
|
416
|
-
@logger = nil
|
|
417
|
-
|
|
418
|
-
def self.logger
|
|
419
|
-
@logger
|
|
420
|
-
end
|
|
421
|
-
|
|
422
534
|
def self.version_check
|
|
423
535
|
# Previously, this method raised an error for versions >= 2.4
|
|
424
536
|
# Now we support versions up to and including 2.4.7
|
|
@@ -507,6 +619,13 @@ module IOStreams
|
|
|
507
619
|
results
|
|
508
620
|
end
|
|
509
621
|
|
|
622
|
+
def self.reject_newlines!(**fields)
|
|
623
|
+
fields.each_pair do |field, value|
|
|
624
|
+
next if value.nil?
|
|
625
|
+
raise(ArgumentError, "IOStreams::Pgp.generate_key: :#{field} cannot contain newlines") if value.to_s =~ /[\r\n]/
|
|
626
|
+
end
|
|
627
|
+
end
|
|
628
|
+
|
|
510
629
|
def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
|
|
511
630
|
keys = private ? "secret-keys" : "keys"
|
|
512
631
|
|
|
@@ -517,9 +636,9 @@ module IOStreams
|
|
|
517
636
|
key_id = key_info[:key_id]
|
|
518
637
|
next unless key_id
|
|
519
638
|
|
|
520
|
-
command = "
|
|
521
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
|
522
|
-
logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
|
|
639
|
+
command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", key_id)
|
|
640
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
641
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}#{out}" }
|
|
523
642
|
|
|
524
643
|
unless status.success?
|
|
525
644
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
|
@@ -532,18 +651,27 @@ module IOStreams
|
|
|
532
651
|
def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
|
|
533
652
|
keys = private ? "secret-keys" : "keys"
|
|
534
653
|
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
654
|
+
# List the fingerprints, then delete each one. Previously this shelled out
|
|
655
|
+
# to a `for` loop, which allowed shell injection via :email / :key_id.
|
|
656
|
+
list_command = gpg_command("--list-#{keys}", "--with-colons", "--fingerprint", (email || key_id).to_s)
|
|
657
|
+
list_out, list_err, = Open3.capture3(*list_command, binmode: true)
|
|
658
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{list_command.shelljoin}\n#{list_err}: #{list_out}" }
|
|
659
|
+
|
|
660
|
+
return false if list_err =~ /(not found|no public key)/i
|
|
661
|
+
|
|
662
|
+
fingerprints = list_out.each_line.select { |line| line.start_with?("fpr") }.map { |line| line.split(":")[9] }.compact
|
|
663
|
+
return false if fingerprints.empty?
|
|
538
664
|
|
|
539
|
-
|
|
540
|
-
|
|
665
|
+
fingerprints.each do |fingerprint|
|
|
666
|
+
command = gpg_command("--batch", "--no-tty", "--yes", "--delete-#{keys}", fingerprint)
|
|
667
|
+
out, err, status = Open3.capture3(*command, binmode: true)
|
|
668
|
+
IOStreams.logger&.debug { "IOStreams::Pgp.delete_keys: #{command.shelljoin}\n#{err}: #{out}" }
|
|
541
669
|
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
670
|
+
unless status.success?
|
|
671
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
|
672
|
+
end
|
|
673
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
|
|
545
674
|
end
|
|
546
|
-
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
|
|
547
675
|
|
|
548
676
|
true
|
|
549
677
|
end
|
data/lib/io_streams/reader.rb
CHANGED
|
@@ -10,13 +10,13 @@ module IOStreams
|
|
|
10
10
|
end
|
|
11
11
|
|
|
12
12
|
# When a Writer supports streams, also allow it to simply support a file
|
|
13
|
-
def self.file(file_name,
|
|
14
|
-
::File.open(file_name, "rb") { |file| stream(file,
|
|
13
|
+
def self.file(file_name, **args, &block)
|
|
14
|
+
::File.open(file_name, "rb") { |file| stream(file, **args, &block) }
|
|
15
15
|
end
|
|
16
16
|
|
|
17
17
|
# For processing by either a file name or an open IO stream.
|
|
18
|
-
def self.open(file_name_or_io, **args, &
|
|
19
|
-
file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &
|
|
18
|
+
def self.open(file_name_or_io, **args, &)
|
|
19
|
+
file_name_or_io.is_a?(String) ? file(file_name_or_io, **args, &) : stream(file_name_or_io, **args, &)
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
attr_reader :input_stream
|
|
@@ -16,7 +16,7 @@ module IOStreams
|
|
|
16
16
|
|
|
17
17
|
# When reading from a file also add the line reader stream
|
|
18
18
|
def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
|
|
19
|
-
IOStreams::Line::Reader.file(file_name,
|
|
19
|
+
IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
|
|
20
20
|
yield new(io, original_file_name: original_file_name, **args)
|
|
21
21
|
end
|
|
22
22
|
end
|
|
@@ -35,11 +35,10 @@ module IOStreams
|
|
|
35
35
|
# format_options: [Hash]
|
|
36
36
|
# Any specialized format specific options. For example, `:fixed` format requires the file definition.
|
|
37
37
|
#
|
|
38
|
-
# columns [Array<String>]
|
|
38
|
+
# columns [Array<String|Symbol>]
|
|
39
39
|
# The header columns when the file does not include a header row.
|
|
40
40
|
# Note:
|
|
41
|
-
#
|
|
42
|
-
# with MongoDB when it converts symbol keys to strings.
|
|
41
|
+
# Column names are converted to strings.
|
|
43
42
|
#
|
|
44
43
|
# allowed_columns [Array<String>]
|
|
45
44
|
# List of columns to allow.
|
|
@@ -18,7 +18,7 @@ module IOStreams
|
|
|
18
18
|
|
|
19
19
|
# When writing to a file also add the line writer stream
|
|
20
20
|
def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
|
|
21
|
-
IOStreams::Line::Writer.file(file_name,
|
|
21
|
+
IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
|
|
22
22
|
yield new(io, original_file_name: original_file_name, **args, &block)
|
|
23
23
|
end
|
|
24
24
|
end
|
|
@@ -37,11 +37,10 @@ module IOStreams
|
|
|
37
37
|
# format_options: [Hash]
|
|
38
38
|
# Any specialized format specific options. For example, `:fixed` format requires the file definition.
|
|
39
39
|
#
|
|
40
|
-
# columns [Array<String>]
|
|
40
|
+
# columns [Array<String|Symbol>]
|
|
41
41
|
# The header columns when the file does not include a header row.
|
|
42
42
|
# Note:
|
|
43
|
-
#
|
|
44
|
-
# with MongoDB when it converts symbol keys to strings.
|
|
43
|
+
# Column names are converted to strings.
|
|
45
44
|
#
|
|
46
45
|
# allowed_columns [Array<String>]
|
|
47
46
|
# List of columns to allow.
|
|
@@ -14,7 +14,7 @@ module IOStreams
|
|
|
14
14
|
|
|
15
15
|
# When reading from a file also add the line reader stream
|
|
16
16
|
def self.file(file_name, original_file_name: file_name, delimiter: $/, **args)
|
|
17
|
-
IOStreams::Line::Reader.file(file_name,
|
|
17
|
+
IOStreams::Line::Reader.file(file_name, delimiter: delimiter) do |io|
|
|
18
18
|
yield new(io, original_file_name: original_file_name, **args)
|
|
19
19
|
end
|
|
20
20
|
end
|
|
@@ -21,7 +21,7 @@ module IOStreams
|
|
|
21
21
|
|
|
22
22
|
# When writing to a file also add the line writer stream
|
|
23
23
|
def self.file(file_name, original_file_name: file_name, delimiter: $/, **args, &block)
|
|
24
|
-
IOStreams::Line::Writer.file(file_name,
|
|
24
|
+
IOStreams::Line::Writer.file(file_name, delimiter: delimiter) do |io|
|
|
25
25
|
yield new(io, original_file_name: original_file_name, **args, &block)
|
|
26
26
|
end
|
|
27
27
|
end
|
data/lib/io_streams/stream.rb
CHANGED
|
@@ -18,8 +18,8 @@ module IOStreams
|
|
|
18
18
|
# Example:
|
|
19
19
|
#
|
|
20
20
|
# IOStreams.path("tempfile2527").stream(:zip).stream(:pgp, passphrase: "receiver_passphrase").read
|
|
21
|
-
def stream(stream, **
|
|
22
|
-
builder.stream(stream, **
|
|
21
|
+
def stream(stream, **)
|
|
22
|
+
builder.stream(stream, **)
|
|
23
23
|
self
|
|
24
24
|
end
|
|
25
25
|
|
|
@@ -33,15 +33,15 @@ module IOStreams
|
|
|
33
33
|
# IOStreams.path("keep_safe.enc").option(:pgp, passphrase: "receiver_passphrase").read
|
|
34
34
|
#
|
|
35
35
|
# IOStreams.path(output_file_name).option(:pgp, passphrase: "receiver_passphrase").read
|
|
36
|
-
def option(stream, **
|
|
37
|
-
builder.option(stream, **
|
|
36
|
+
def option(stream, **)
|
|
37
|
+
builder.option(stream, **)
|
|
38
38
|
self
|
|
39
39
|
end
|
|
40
40
|
|
|
41
41
|
# Adds the options for the specified stream as an option,
|
|
42
42
|
# but if streams have already been added it is instead added as a stream.
|
|
43
|
-
def option_or_stream(stream, **
|
|
44
|
-
builder.option_or_stream(stream, **
|
|
43
|
+
def option_or_stream(stream, **)
|
|
44
|
+
builder.option_or_stream(stream, **)
|
|
45
45
|
self
|
|
46
46
|
end
|
|
47
47
|
|
|
@@ -93,21 +93,26 @@ module IOStreams
|
|
|
93
93
|
def each(mode = :line, **args, &block)
|
|
94
94
|
raise(ArgumentError, "Invalid mode: #{mode.inspect}") if mode == :stream
|
|
95
95
|
|
|
96
|
-
#
|
|
96
|
+
# Deliberately not returning an Enumerator when no block is given.
|
|
97
|
+
# The stream pipeline manages resources via block scope: every stream is opened with an
|
|
98
|
+
# `ensure` that closes the file handle, reaps the gpg subprocess, deletes temp files, etc.
|
|
99
|
+
# A Fiber-backed Enumerator (e.g. `to_enum(__method__, mode, **args)`) would leave that block
|
|
100
|
+
# suspended; if the caller abandons a partially-consumed enumerator, none of the cleanup runs
|
|
101
|
+
# until GC collects the Fiber, leaking file descriptors, gpg processes, and temp files.
|
|
97
102
|
reader(mode, **args) { |stream| stream.each(&block) }
|
|
98
103
|
end
|
|
99
104
|
|
|
100
105
|
# Returns a Reader for reading a file / stream
|
|
101
|
-
def reader(mode = :stream, **args, &
|
|
106
|
+
def reader(mode = :stream, **args, &)
|
|
102
107
|
case mode
|
|
103
108
|
when :stream
|
|
104
|
-
stream_reader(&
|
|
109
|
+
stream_reader(&)
|
|
105
110
|
when :line
|
|
106
|
-
line_reader(**args, &
|
|
111
|
+
line_reader(**args, &)
|
|
107
112
|
when :array
|
|
108
|
-
row_reader(**args, &
|
|
113
|
+
row_reader(**args, &)
|
|
109
114
|
when :hash
|
|
110
|
-
record_reader(**args, &
|
|
115
|
+
record_reader(**args, &)
|
|
111
116
|
else
|
|
112
117
|
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
|
113
118
|
end
|
|
@@ -124,16 +129,16 @@ module IOStreams
|
|
|
124
129
|
end
|
|
125
130
|
|
|
126
131
|
# Returns a Writer for writing to a file / stream
|
|
127
|
-
def writer(mode = :stream, **args, &
|
|
132
|
+
def writer(mode = :stream, **args, &)
|
|
128
133
|
case mode
|
|
129
134
|
when :stream
|
|
130
|
-
stream_writer(&
|
|
135
|
+
stream_writer(&)
|
|
131
136
|
when :line
|
|
132
|
-
line_writer(**args, &
|
|
137
|
+
line_writer(**args, &)
|
|
133
138
|
when :array
|
|
134
|
-
row_writer(**args, &
|
|
139
|
+
row_writer(**args, &)
|
|
135
140
|
when :hash
|
|
136
|
-
record_writer(**args, &
|
|
141
|
+
record_writer(**args, &)
|
|
137
142
|
else
|
|
138
143
|
raise(ArgumentError, "Invalid mode: #{mode.inspect}")
|
|
139
144
|
end
|
|
@@ -171,7 +176,7 @@ module IOStreams
|
|
|
171
176
|
# IOStreams.path("target_file.json").copy_from("source_file_name.csv.gz", convert: false)
|
|
172
177
|
#
|
|
173
178
|
# # Advanced copy with custom stream conversions on source and target.
|
|
174
|
-
# source = IOStreams.path("source_file").stream(encoding: "BINARY")
|
|
179
|
+
# source = IOStreams.path("source_file").stream(:encode, encoding: "BINARY")
|
|
175
180
|
# IOStreams.path("target_file.pgp").option(:pgp, passphrase: "hello").copy_from(source)
|
|
176
181
|
def copy_from(source, convert: true, mode: nil, **args)
|
|
177
182
|
if convert
|
|
@@ -322,18 +327,19 @@ module IOStreams
|
|
|
322
327
|
@builder ||= IOStreams::Builder.new
|
|
323
328
|
end
|
|
324
329
|
|
|
325
|
-
def stream_reader(&
|
|
326
|
-
builder.reader(io_stream, &
|
|
330
|
+
def stream_reader(&)
|
|
331
|
+
builder.reader(io_stream, &)
|
|
327
332
|
end
|
|
328
333
|
|
|
329
334
|
def line_reader(embedded_within: nil, **args)
|
|
330
335
|
embedded_within = '"' if embedded_within.nil? && builder.file_name&.include?(".csv")
|
|
331
336
|
|
|
332
337
|
stream_reader do |io|
|
|
333
|
-
yield IOStreams::Line::Reader.new(
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
338
|
+
yield IOStreams::Line::Reader.new(
|
|
339
|
+
io,
|
|
340
|
+
embedded_within: embedded_within,
|
|
341
|
+
**args
|
|
342
|
+
)
|
|
337
343
|
end
|
|
338
344
|
end
|
|
339
345
|
|
|
@@ -363,20 +369,20 @@ module IOStreams
|
|
|
363
369
|
end
|
|
364
370
|
end
|
|
365
371
|
|
|
366
|
-
def stream_writer(&
|
|
367
|
-
builder.writer(io_stream, &
|
|
372
|
+
def stream_writer(&)
|
|
373
|
+
builder.writer(io_stream, &)
|
|
368
374
|
end
|
|
369
375
|
|
|
370
376
|
def line_writer(**args, &block)
|
|
371
|
-
return block.call(io_stream) if io_stream
|
|
377
|
+
return block.call(io_stream) if io_stream.is_a?(IOStreams::Line::Writer)
|
|
372
378
|
|
|
373
379
|
writer do |io|
|
|
374
|
-
IOStreams::Line::Writer.stream(io,
|
|
380
|
+
IOStreams::Line::Writer.stream(io, **args, &block)
|
|
375
381
|
end
|
|
376
382
|
end
|
|
377
383
|
|
|
378
384
|
def row_writer(delimiter: $/, **args, &block)
|
|
379
|
-
return block.call(io_stream) if io_stream
|
|
385
|
+
return block.call(io_stream) if io_stream.is_a?(IOStreams::Row::Writer)
|
|
380
386
|
|
|
381
387
|
line_writer(delimiter: delimiter) do |io|
|
|
382
388
|
IOStreams::Row::Writer.stream(
|
|
@@ -391,7 +397,7 @@ module IOStreams
|
|
|
391
397
|
end
|
|
392
398
|
|
|
393
399
|
def record_writer(delimiter: $/, **args, &block)
|
|
394
|
-
return block.call(io_stream) if io_stream
|
|
400
|
+
return block.call(io_stream) if io_stream.is_a?(IOStreams::Record::Writer)
|
|
395
401
|
|
|
396
402
|
line_writer(delimiter: delimiter) do |io|
|
|
397
403
|
IOStreams::Record::Writer.stream(
|
|
@@ -2,10 +2,10 @@ module IOStreams
|
|
|
2
2
|
module SymmetricEncryption
|
|
3
3
|
class Reader < IOStreams::Reader
|
|
4
4
|
# read from a file/stream using Symmetric Encryption
|
|
5
|
-
def self.stream(input_stream, **args, &
|
|
5
|
+
def self.stream(input_stream, **args, &)
|
|
6
6
|
Utils.load_soft_dependency("symmetric-encryption", ".enc streaming") unless defined?(SymmetricEncryption)
|
|
7
7
|
|
|
8
|
-
::SymmetricEncryption::Reader.open(input_stream, **args, &
|
|
8
|
+
::SymmetricEncryption::Reader.open(input_stream, **args, &)
|
|
9
9
|
end
|
|
10
10
|
end
|
|
11
11
|
end
|