iostreams 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -3
- data/lib/io_streams/builder.rb +9 -6
- data/lib/io_streams/bzip2/reader.rb +5 -3
- data/lib/io_streams/bzip2/writer.rb +5 -3
- data/lib/io_streams/deprecated.rb +1 -1
- data/lib/io_streams/encode/reader.rb +1 -1
- data/lib/io_streams/encode/writer.rb +1 -1
- data/lib/io_streams/errors.rb +10 -0
- data/lib/io_streams/io_streams.rb +1 -1
- data/lib/io_streams/paths/file.rb +4 -4
- data/lib/io_streams/paths/http.rb +6 -3
- data/lib/io_streams/paths/s3.rb +27 -8
- data/lib/io_streams/paths/sftp.rb +16 -4
- data/lib/io_streams/pgp.rb +80 -61
- data/lib/io_streams/stream.rb +20 -9
- data/lib/io_streams/tabular.rb +5 -2
- data/lib/io_streams/tabular/header.rb +14 -12
- data/lib/io_streams/tabular/parser/fixed.rb +135 -25
- data/lib/io_streams/utils.rb +4 -4
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/zip/reader.rb +1 -1
- data/test/bzip2_writer_test.rb +6 -4
- data/test/io_streams_test.rb +2 -2
- data/test/paths/file_test.rb +1 -1
- data/test/paths/s3_test.rb +3 -3
- data/test/paths/sftp_test.rb +4 -4
- data/test/pgp_test.rb +54 -4
- data/test/pgp_writer_test.rb +3 -3
- data/test/tabular_test.rb +55 -26
- data/test/test_helper.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 36a51f41f33a1c58f1d5d03ed857784e86de4588ca38aa5a596e479f8661faf3
|
4
|
+
data.tar.gz: 23dd967c21581520675799692296bb901e2dc539f008fa30623f4bd72a2c90f3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 81d5f7ea50b7a5b06a5d26b758eb0639b0c820735cd0cf1545c6298f244b96b0070d3f8a11deed0ef8398b5b6c461561b3c43661612e3eeda19bdf61abb549ab
|
7
|
+
data.tar.gz: 19f6f2051351533029cd7fca3cd6e7be3a4e5d3ea4ff2ae512b0b98a12b84639ae63e6a4decc0ff3edf1d50d5c14ddc1b869767f2f6c8793fc392f6ad02d939f
|
data/README.md
CHANGED
@@ -1,7 +1,8 @@
|
|
1
|
-
#
|
1
|
+
# IOStreams
|
2
2
|
[](https://rubygems.org/gems/iostreams) [](https://travis-ci.org/rocketjob/iostreams) [](https://rubygems.org/gems/iostreams) [](http://opensource.org/licenses/Apache-2.0)  [-Support-brightgreen.svg)](https://gitter.im/rocketjob/support)
|
3
3
|
|
4
|
-
|
4
|
+
IOStreams is an incredibly powerful streaming library that makes changes to file formats, compression, encryption,
|
5
|
+
or storage mechanism transparent to the application.
|
5
6
|
|
6
7
|
## Project Status
|
7
8
|
|
@@ -9,7 +10,9 @@ Production Ready, heavily used in production environments, many as part of Rocke
|
|
9
10
|
|
10
11
|
## Documentation
|
11
12
|
|
12
|
-
[
|
13
|
+
Start with the [IOStreams tutorial](https://iostreams.rocketjob.io/tutorial) to get a great introduction to IOStreams.
|
14
|
+
|
15
|
+
Next, checkout the remaining [IOStreams documentation](https://iostreams.rocketjob.io/)
|
13
16
|
|
14
17
|
## Versioning
|
15
18
|
|
data/lib/io_streams/builder.rb
CHANGED
@@ -20,7 +20,7 @@ module IOStreams
|
|
20
20
|
raise(ArgumentError, "Cannot call #option unless the `file_name` was already set}") unless file_name
|
21
21
|
|
22
22
|
@options ||= {}
|
23
|
-
if opts = @options[stream]
|
23
|
+
if (opts = @options[stream])
|
24
24
|
opts.merge!(options)
|
25
25
|
else
|
26
26
|
@options[stream] = options.dup
|
@@ -40,7 +40,7 @@ module IOStreams
|
|
40
40
|
raise(ArgumentError, "Invalid stream: #{stream.inspect}") unless IOStreams.extensions.include?(stream)
|
41
41
|
|
42
42
|
@streams ||= {}
|
43
|
-
if opts = @streams[stream]
|
43
|
+
if (opts = @streams[stream])
|
44
44
|
opts.merge!(options)
|
45
45
|
else
|
46
46
|
@streams[stream] = options.dup
|
@@ -91,7 +91,8 @@ module IOStreams
|
|
91
91
|
private
|
92
92
|
|
93
93
|
def class_for_stream(type, stream)
|
94
|
-
ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
|
94
|
+
ext = IOStreams.extensions[stream.nil? ? nil : stream.to_sym] ||
|
95
|
+
raise(ArgumentError, "Unknown Stream type: #{stream.inspect}")
|
95
96
|
ext.send("#{type}_class") || raise(ArgumentError, "No #{type} registered for Stream type: #{stream.inspect}")
|
96
97
|
end
|
97
98
|
|
@@ -99,7 +100,7 @@ module IOStreams
|
|
99
100
|
def parse_extensions
|
100
101
|
parts = ::File.basename(file_name).split(".")
|
101
102
|
extensions = []
|
102
|
-
while extension = parts.pop
|
103
|
+
while (extension = parts.pop)
|
103
104
|
sym = extension.downcase.to_sym
|
104
105
|
break unless IOStreams.extensions[sym]
|
105
106
|
|
@@ -116,10 +117,12 @@ module IOStreams
|
|
116
117
|
block.call(io_stream)
|
117
118
|
elsif pipeline.size == 1
|
118
119
|
stream, opts = pipeline.first
|
119
|
-
class_for_stream(type, stream).open(io_stream, opts, &block)
|
120
|
+
class_for_stream(type, stream).open(io_stream, **opts, &block)
|
120
121
|
else
|
121
122
|
# Daisy chain multiple streams together
|
122
|
-
last = pipeline.keys.inject(block)
|
123
|
+
last = pipeline.keys.inject(block) do |inner, stream_sym|
|
124
|
+
->(io) { class_for_stream(type, stream_sym).open(io, **pipeline[stream_sym], &inner) }
|
125
|
+
end
|
123
126
|
last.call(io_stream)
|
124
127
|
end
|
125
128
|
end
|
@@ -2,11 +2,13 @@ module IOStreams
|
|
2
2
|
module Bzip2
|
3
3
|
class Reader < IOStreams::Reader
|
4
4
|
# Read from a Bzip2 stream, decompressing the contents as it is read
|
5
|
-
def self.stream(input_stream, **
|
6
|
-
|
5
|
+
def self.stream(input_stream, **args)
|
6
|
+
unless defined?(::Bzip2::FFI)
|
7
|
+
Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
|
8
|
+
end
|
7
9
|
|
8
10
|
begin
|
9
|
-
io =
|
11
|
+
io = ::Bzip2::FFI::Reader.new(input_stream, args)
|
10
12
|
yield io
|
11
13
|
ensure
|
12
14
|
io&.close
|
@@ -2,11 +2,13 @@ module IOStreams
|
|
2
2
|
module Bzip2
|
3
3
|
class Writer < IOStreams::Writer
|
4
4
|
# Write to a stream, compressing with Bzip2
|
5
|
-
def self.stream(input_stream, original_file_name: nil, **
|
6
|
-
|
5
|
+
def self.stream(input_stream, original_file_name: nil, **args)
|
6
|
+
unless defined?(::Bzip2::FFI)
|
7
|
+
Utils.load_soft_dependency("bzip2-ffi", "Bzip2", "bzip2/ffi")
|
8
|
+
end
|
7
9
|
|
8
10
|
begin
|
9
|
-
io =
|
11
|
+
io = ::Bzip2::FFI::Writer.new(input_stream, args)
|
10
12
|
yield io
|
11
13
|
ensure
|
12
14
|
io&.close
|
@@ -206,7 +206,7 @@ module IOStreams
|
|
206
206
|
elsif streams.is_a?(Array)
|
207
207
|
streams.each { |stream| apply_old_style_streams(path, stream) }
|
208
208
|
elsif streams.is_a?(Hash)
|
209
|
-
streams.each_pair { |stream, options| path.stream(stream, options) }
|
209
|
+
streams.each_pair { |stream, options| path.stream(stream, **options) }
|
210
210
|
else
|
211
211
|
raise ArgumentError, "Invalid old style stream supplied: #{params.inspect}"
|
212
212
|
end
|
@@ -73,7 +73,7 @@ module IOStreams
|
|
73
73
|
# EOF reached?
|
74
74
|
return unless block
|
75
75
|
|
76
|
-
block = block.encode(@encoding,
|
76
|
+
block = block.encode(@encoding, **@encoding_options) unless block.encoding == @encoding
|
77
77
|
block = @cleaner.call(block, @replace) if @cleaner
|
78
78
|
block
|
79
79
|
end
|
@@ -66,7 +66,7 @@ module IOStreams
|
|
66
66
|
return 0 if data.nil?
|
67
67
|
|
68
68
|
data = data.to_s
|
69
|
-
block = data.encoding == @encoding ? data : data.encode(@encoding,
|
69
|
+
block = data.encoding == @encoding ? data : data.encode(@encoding, **@encoding_options)
|
70
70
|
block = @cleaner.call(block, @replace) if @cleaner
|
71
71
|
@output_stream.write(block)
|
72
72
|
end
|
data/lib/io_streams/errors.rb
CHANGED
@@ -18,5 +18,15 @@ module IOStreams
|
|
18
18
|
# When the specified delimiter is not found in the supplied stream / file
|
19
19
|
class DelimiterNotFound < Error
|
20
20
|
end
|
21
|
+
|
22
|
+
# Fixed length line has the wrong length
|
23
|
+
class InvalidLineLength < Error
|
24
|
+
end
|
25
|
+
|
26
|
+
class ValueTooLong < Error
|
27
|
+
end
|
28
|
+
|
29
|
+
class InvalidLayout < Error
|
30
|
+
end
|
21
31
|
end
|
22
32
|
end
|
@@ -58,7 +58,7 @@ module IOStreams
|
|
58
58
|
end
|
59
59
|
|
60
60
|
# For an existing IO Stream
|
61
|
-
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').
|
61
|
+
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').read
|
62
62
|
# IOStreams.stream(io).file_name('blah.zip').encoding('BINARY').each(:line){ ... }
|
63
63
|
# IOStreams.stream(io).file_name('blah.csv.zip').each(:line) { ... }
|
64
64
|
# IOStreams.stream(io).stream(:zip).stream(:pgp, passphrase: 'receiver_passphrase').read
|
@@ -15,16 +15,16 @@ module IOStreams
|
|
15
15
|
# Examples:
|
16
16
|
#
|
17
17
|
# # Case Insensitive file name lookup:
|
18
|
-
# IOStreams
|
18
|
+
# IOStreams.path("ruby").glob("r*.md") { |name| puts name }
|
19
19
|
#
|
20
20
|
# # Case Sensitive file name lookup:
|
21
|
-
# IOStreams
|
21
|
+
# IOStreams.path("ruby").each("R*.md", case_sensitive: true) { |name| puts name }
|
22
22
|
#
|
23
23
|
# # Also return the names of directories found during the search:
|
24
|
-
# IOStreams
|
24
|
+
# IOStreams.path("ruby").each("R*.md", directories: true) { |name| puts name }
|
25
25
|
#
|
26
26
|
# # Case Insensitive recursive file name lookup:
|
27
|
-
# IOStreams
|
27
|
+
# IOStreams.path("ruby").glob("**/*.md") { |name| puts name }
|
28
28
|
#
|
29
29
|
# Parameters:
|
30
30
|
# pattern [String]
|
@@ -26,16 +26,19 @@ module IOStreams
|
|
26
26
|
#
|
27
27
|
# http_redirect_count: [Integer]
|
28
28
|
# Maximum number of http redirects to follow.
|
29
|
-
def initialize(url, username: nil, password: nil, http_redirect_count: 10)
|
29
|
+
def initialize(url, username: nil, password: nil, http_redirect_count: 10, parameters: nil)
|
30
30
|
uri = URI.parse(url)
|
31
31
|
unless %w[http https].include?(uri.scheme)
|
32
|
-
raise(
|
32
|
+
raise(
|
33
|
+
ArgumentError,
|
34
|
+
"Invalid URL. Required Format: 'http://<host_name>/<file_name>', or 'https://<host_name>/<file_name>'"
|
35
|
+
)
|
33
36
|
end
|
34
37
|
|
35
38
|
@username = username || uri.user
|
36
39
|
@password = password || uri.password
|
37
40
|
@http_redirect_count = http_redirect_count
|
38
|
-
@url = url
|
41
|
+
@url = parameters ? "#{url}?#{URI.encode_www_form(parameters)}" : url
|
39
42
|
super(uri.path)
|
40
43
|
end
|
41
44
|
|
data/lib/io_streams/paths/s3.rb
CHANGED
@@ -3,7 +3,7 @@ require "uri"
|
|
3
3
|
module IOStreams
|
4
4
|
module Paths
|
5
5
|
class S3 < IOStreams::Path
|
6
|
-
attr_reader :bucket_name, :client
|
6
|
+
attr_reader :bucket_name, :client, :options
|
7
7
|
|
8
8
|
# Arguments:
|
9
9
|
#
|
@@ -92,7 +92,7 @@ module IOStreams
|
|
92
92
|
# encrypting data. This value is used to store the object and then it is
|
93
93
|
# discarded; Amazon does not store the encryption key. The key must be
|
94
94
|
# appropriate for use with the algorithm specified in the
|
95
|
-
# x-amz-server-side
|
95
|
+
# x-amz-server-side-encryption-customer-algorithm header.
|
96
96
|
#
|
97
97
|
# @option params [String] :sse_customer_key_md5
|
98
98
|
# Specifies the 128-bit MD5 digest of the encryption key according to
|
@@ -179,17 +179,36 @@ module IOStreams
|
|
179
179
|
#
|
180
180
|
# Notes:
|
181
181
|
# - Can copy across buckets.
|
182
|
+
# - No stream conversions are applied.
|
182
183
|
def move_to(target_path)
|
184
|
+
target = copy_to(target_path, convert: false)
|
185
|
+
delete
|
186
|
+
target
|
187
|
+
end
|
188
|
+
|
189
|
+
# Make S3 perform direct copies within S3 itself.
|
190
|
+
def copy_to(target_path, convert: true)
|
191
|
+
return super(target_path) if convert
|
192
|
+
|
183
193
|
target = IOStreams.new(target_path)
|
184
194
|
return super(target) unless target.is_a?(self.class)
|
185
195
|
|
186
196
|
source_name = ::File.join(bucket_name, path)
|
187
|
-
|
188
|
-
client.copy_object(bucket: target.bucket_name, key: target.path, copy_source: source_name)
|
189
|
-
delete
|
197
|
+
client.copy_object(options.merge(bucket: target.bucket_name, key: target.path, copy_source: source_name))
|
190
198
|
target
|
191
199
|
end
|
192
200
|
|
201
|
+
# Make S3 perform direct copies within S3 itself.
|
202
|
+
def copy_from(source_path, convert: true)
|
203
|
+
return super(source_path) if convert
|
204
|
+
|
205
|
+
source = IOStreams.new(source_path)
|
206
|
+
return super(source, **args) unless source.is_a?(self.class)
|
207
|
+
|
208
|
+
source_name = ::File.join(source.bucket_name, source.path)
|
209
|
+
client.copy_object(options.merge(bucket: bucket_name, key: path, copy_source: source_name))
|
210
|
+
end
|
211
|
+
|
193
212
|
# S3 logically creates paths when a key is set.
|
194
213
|
def mkpath
|
195
214
|
self
|
@@ -220,7 +239,7 @@ module IOStreams
|
|
220
239
|
# Shortcut method if caller has a filename already with no other streams applied:
|
221
240
|
def read_file(file_name)
|
222
241
|
::File.open(file_name, "wb") do |file|
|
223
|
-
client.get_object(
|
242
|
+
client.get_object(options.merge(response_target: file, bucket: bucket_name, key: path))
|
224
243
|
end
|
225
244
|
end
|
226
245
|
|
@@ -248,10 +267,10 @@ module IOStreams
|
|
248
267
|
# Use multipart file upload
|
249
268
|
s3 = Aws::S3::Resource.new(client: client)
|
250
269
|
obj = s3.bucket(bucket_name).object(path)
|
251
|
-
obj.upload_file(file_name)
|
270
|
+
obj.upload_file(file_name, options)
|
252
271
|
else
|
253
272
|
::File.open(file_name, "rb") do |file|
|
254
|
-
client.put_object(
|
273
|
+
client.put_object(options.merge(bucket: bucket_name, key: path, body: file))
|
255
274
|
end
|
256
275
|
end
|
257
276
|
end
|
@@ -173,7 +173,10 @@ module IOStreams
|
|
173
173
|
writer.close
|
174
174
|
out = reader.read.chomp
|
175
175
|
unless waith_thr.value.success?
|
176
|
-
raise(
|
176
|
+
raise(
|
177
|
+
Errors::CommunicationsFailure,
|
178
|
+
"Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
179
|
+
)
|
177
180
|
end
|
178
181
|
|
179
182
|
out
|
@@ -183,7 +186,10 @@ module IOStreams
|
|
183
186
|
rescue StandardError
|
184
187
|
nil
|
185
188
|
end
|
186
|
-
raise(
|
189
|
+
raise(
|
190
|
+
Errors::CommunicationsFailure,
|
191
|
+
"Download failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
192
|
+
)
|
187
193
|
end
|
188
194
|
end
|
189
195
|
end
|
@@ -201,7 +207,10 @@ module IOStreams
|
|
201
207
|
writer.close
|
202
208
|
out = reader.read.chomp
|
203
209
|
unless waith_thr.value.success?
|
204
|
-
raise(
|
210
|
+
raise(
|
211
|
+
Errors::CommunicationsFailure,
|
212
|
+
"Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
213
|
+
)
|
205
214
|
end
|
206
215
|
|
207
216
|
out
|
@@ -211,7 +220,10 @@ module IOStreams
|
|
211
220
|
rescue StandardError
|
212
221
|
nil
|
213
222
|
end
|
214
|
-
raise(
|
223
|
+
raise(
|
224
|
+
Errors::CommunicationsFailure,
|
225
|
+
"Upload failed calling #{self.class.sftp_bin} via #{self.class.sshpass_bin}: #{out}"
|
226
|
+
)
|
215
227
|
end
|
216
228
|
end
|
217
229
|
end
|
data/lib/io_streams/pgp.rb
CHANGED
@@ -46,7 +46,15 @@ module IOStreams
|
|
46
46
|
# `SecureRandom.urlsafe_base64(128)`
|
47
47
|
#
|
48
48
|
# See `man gpg` for the remaining options
|
49
|
-
def self.generate_key(name:,
|
49
|
+
def self.generate_key(name:,
|
50
|
+
email:,
|
51
|
+
comment: nil,
|
52
|
+
passphrase:,
|
53
|
+
key_type: "RSA",
|
54
|
+
key_length: 4096,
|
55
|
+
subkey_type: "RSA",
|
56
|
+
subkey_length: key_length,
|
57
|
+
expire_date: nil)
|
50
58
|
version_check
|
51
59
|
params = ""
|
52
60
|
params << "Key-Type: #{key_type}\n" if key_type
|
@@ -63,12 +71,11 @@ module IOStreams
|
|
63
71
|
|
64
72
|
out, err, status = Open3.capture3(command, binmode: true, stdin_data: params)
|
65
73
|
logger&.debug { "IOStreams::Pgp.generate_key: #{command}\n#{params}\n#{err}#{out}" }
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}")
|
74
|
+
|
75
|
+
raise(Pgp::Failure, "GPG Failed to generate key: #{err}#{out}") unless status.success?
|
76
|
+
|
77
|
+
if (match = err.match(/gpg: key ([0-9A-F]+)\s+/))
|
78
|
+
match[1]
|
72
79
|
end
|
73
80
|
end
|
74
81
|
|
@@ -77,7 +84,8 @@ module IOStreams
|
|
77
84
|
# Returns false if no key was found.
|
78
85
|
# Raises an exception if it fails to delete the key.
|
79
86
|
#
|
80
|
-
# email: [String]
|
87
|
+
# email: [String] Optional email address for the key.
|
88
|
+
# key_id: [String] Optional id for the key.
|
81
89
|
#
|
82
90
|
# public: [true|false]
|
83
91
|
# Whether to delete the public key
|
@@ -86,12 +94,12 @@ module IOStreams
|
|
86
94
|
# private: [true|false]
|
87
95
|
# Whether to delete the private key
|
88
96
|
# Default: false
|
89
|
-
def self.delete_keys(email
|
97
|
+
def self.delete_keys(email: nil, key_id: nil, public: true, private: false)
|
90
98
|
version_check
|
91
99
|
method_name = pgp_version.to_f >= 2.2 ? :delete_public_or_private_keys : :delete_public_or_private_keys_v1
|
92
100
|
status = false
|
93
|
-
status = send(method_name, email: email, private: true) if private
|
94
|
-
status = send(method_name, email: email, private: false) if public
|
101
|
+
status = send(method_name, email: email, key_id: key_id, private: true) if private
|
102
|
+
status = send(method_name, email: email, key_id: key_id, private: false) if public
|
95
103
|
status
|
96
104
|
end
|
97
105
|
|
@@ -150,16 +158,15 @@ module IOStreams
|
|
150
158
|
|
151
159
|
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
152
160
|
logger&.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" }
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
end
|
161
|
+
|
162
|
+
raise(Pgp::Failure, "GPG Failed extracting key details: #{err} #{out}") unless status.success? && out.length.positive?
|
163
|
+
|
164
|
+
# Sample Output:
|
165
|
+
#
|
166
|
+
# pub 4096R/3A5456F5 2017-06-07
|
167
|
+
# uid Joe Bloggs <j@bloggs.net>
|
168
|
+
# sub 4096R/2C9B240B 2017-06-07
|
169
|
+
parse_list_output(out)
|
163
170
|
end
|
164
171
|
|
165
172
|
# Returns [String] containing all the public keys for the supplied email address.
|
@@ -181,11 +188,10 @@ module IOStreams
|
|
181
188
|
|
182
189
|
out, err, status = Open3.capture3(command, binmode: true)
|
183
190
|
logger&.debug { "IOStreams::Pgp.export: #{command}\n#{err}" }
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
end
|
191
|
+
|
192
|
+
raise(Pgp::Failure, "GPG Failed reading key: #{email}: #{err}") unless status.success? && out.length.positive?
|
193
|
+
|
194
|
+
out
|
189
195
|
end
|
190
196
|
|
191
197
|
# Imports the supplied public/private key
|
@@ -251,11 +257,14 @@ module IOStreams
|
|
251
257
|
def self.import_and_trust(key:)
|
252
258
|
raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == "")
|
253
259
|
|
254
|
-
|
255
|
-
|
260
|
+
key_info = key_info(key: key).last
|
261
|
+
|
262
|
+
email = key_info.fetch(:email, nil)
|
263
|
+
key_id = key_info.fetch(:key_id, nil)
|
264
|
+
raise(ArgumentError, "Recipient email or key id cannot be extracted from supplied key") unless email || key_id
|
256
265
|
|
257
266
|
import(key: key)
|
258
|
-
set_trust(email: email)
|
267
|
+
set_trust(email: email, key_id: key_id)
|
259
268
|
email
|
260
269
|
end
|
261
270
|
|
@@ -266,20 +275,19 @@ module IOStreams
|
|
266
275
|
#
|
267
276
|
# After importing keys, they are not trusted and the relevant trust level must be set.
|
268
277
|
# Default: 5 : Ultimate
|
269
|
-
def self.set_trust(email
|
278
|
+
def self.set_trust(email: nil, key_id: nil, level: 5)
|
270
279
|
version_check
|
271
|
-
fingerprint = fingerprint(email: email)
|
280
|
+
fingerprint = key_id || fingerprint(email: email)
|
272
281
|
return unless fingerprint
|
273
282
|
|
274
283
|
command = "#{executable} --import-ownertrust"
|
275
284
|
trust = "#{fingerprint}:#{level + 1}:\n"
|
276
285
|
out, err, status = Open3.capture3(command, stdin_data: trust)
|
277
286
|
logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
end
|
287
|
+
|
288
|
+
raise(Pgp::Failure, "GPG Failed trusting key: #{err} #{out}") unless status.success?
|
289
|
+
|
290
|
+
err
|
283
291
|
end
|
284
292
|
|
285
293
|
# DEPRECATED - Use key_ids instead of fingerprints
|
@@ -287,18 +295,18 @@ module IOStreams
|
|
287
295
|
version_check
|
288
296
|
Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
|
289
297
|
output = out.read.chomp
|
290
|
-
|
291
|
-
output
|
292
|
-
|
293
|
-
return match[1]
|
294
|
-
end
|
298
|
+
unless waith_thr.value.success?
|
299
|
+
unless output =~ /(public key not found|No public key)/i
|
300
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
|
295
301
|
end
|
296
|
-
|
297
|
-
else
|
298
|
-
return if output =~ /(public key not found|No public key)/i
|
302
|
+
end
|
299
303
|
|
300
|
-
|
304
|
+
output.each_line do |line|
|
305
|
+
if (match = line.match(/\Afpr.*::([^\:]*):\Z/))
|
306
|
+
return match[1]
|
307
|
+
end
|
301
308
|
end
|
309
|
+
nil
|
302
310
|
end
|
303
311
|
end
|
304
312
|
|
@@ -328,7 +336,7 @@ module IOStreams
|
|
328
336
|
# CAMELLIA128, CAMELLIA192, CAMELLIA256
|
329
337
|
# Hash: MD5, SHA1, RIPEMD160, SHA256, SHA384, SHA512, SHA224
|
330
338
|
# Compression: Uncompressed, ZIP, ZLIB, BZIP2
|
331
|
-
if match = out.lines.first.match(/(\d+\.\d+.\d+)/)
|
339
|
+
if (match = out.lines.first.match(/(\d+\.\d+.\d+)/))
|
332
340
|
match[1]
|
333
341
|
end
|
334
342
|
else
|
@@ -348,9 +356,12 @@ module IOStreams
|
|
348
356
|
end
|
349
357
|
|
350
358
|
def self.version_check
|
351
|
-
|
352
|
-
|
353
|
-
|
359
|
+
return unless pgp_version.to_f >= 2.3
|
360
|
+
|
361
|
+
raise(
|
362
|
+
Pgp::UnsupportedVersion,
|
363
|
+
"Version #{pgp_version} of #{executable} is not yet supported. Please submit a Pull Request to support it."
|
364
|
+
)
|
354
365
|
end
|
355
366
|
|
356
367
|
# v2.2.1 output:
|
@@ -370,7 +381,7 @@ module IOStreams
|
|
370
381
|
results = []
|
371
382
|
hash = {}
|
372
383
|
out.each_line do |line|
|
373
|
-
if match = line.match(/(pub|sec)\s+(\D+)(\d+)\s+(\d+-\d+-\d+)\s+(.*)/)
|
384
|
+
if (match = line.match(/(pub|sec)\s+(\D+)(\d+)\s+(\d+-\d+-\d+)\s+(.*)/))
|
374
385
|
# v2.2: pub rsa1024 2017-10-24 [SCEA]
|
375
386
|
hash = {
|
376
387
|
private: match[1] == "sec",
|
@@ -382,7 +393,7 @@ module IOStreams
|
|
382
393
|
match[4]
|
383
394
|
end)
|
384
395
|
}
|
385
|
-
elsif match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?})
|
396
|
+
elsif (match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?}))
|
386
397
|
# Matches: pub 2048R/C7F9D9CB 2016-10-26
|
387
398
|
# Or: pub 2048R/C7F9D9CB 2016-10-26 Receiver <receiver@example.org>
|
388
399
|
hash = {
|
@@ -403,7 +414,7 @@ module IOStreams
|
|
403
414
|
results << hash
|
404
415
|
hash = {}
|
405
416
|
end
|
406
|
-
elsif match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)<(.+)>/)
|
417
|
+
elsif (match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)<(.+)>/))
|
407
418
|
# Matches: uid [ unknown] Joe Bloggs <j@bloggs.net>
|
408
419
|
# Or: uid Joe Bloggs <j@bloggs.net>
|
409
420
|
# v2.2: uid [ultimate] Joe Bloggs <pgp_test@iostreams.net>
|
@@ -412,7 +423,15 @@ module IOStreams
|
|
412
423
|
hash[:trust] = match[2].to_s.strip if match[1]
|
413
424
|
results << hash
|
414
425
|
hash = {}
|
415
|
-
elsif match = line.match(/([
|
426
|
+
elsif (match = line.match(/uid\s+(\[(.+)\]\s+)?(.+)/))
|
427
|
+
# Matches: uid [ unknown] Joe Bloggs
|
428
|
+
# Or: uid Joe Bloggs
|
429
|
+
# v2.2: uid [ultimate] Joe Bloggs
|
430
|
+
hash[:name] = match[3].to_s.strip
|
431
|
+
hash[:trust] = match[2].to_s.strip if match[1]
|
432
|
+
results << hash
|
433
|
+
hash = {}
|
434
|
+
elsif (match = line.match(/([A-Z0-9]+)/))
|
416
435
|
# v2.2 18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
|
417
436
|
hash[:key_id] ||= match[1]
|
418
437
|
end
|
@@ -420,10 +439,10 @@ module IOStreams
|
|
420
439
|
results
|
421
440
|
end
|
422
441
|
|
423
|
-
def self.delete_public_or_private_keys(email
|
442
|
+
def self.delete_public_or_private_keys(email: nil, key_id: nil, private: false)
|
424
443
|
keys = private ? "secret-keys" : "keys"
|
425
444
|
|
426
|
-
list = list_keys(email: email, private: private)
|
445
|
+
list = email ? list_keys(email: email, private: private) : list_keys(key_id: key_id)
|
427
446
|
return false if list.empty?
|
428
447
|
|
429
448
|
list.each do |key_info|
|
@@ -435,17 +454,17 @@ module IOStreams
|
|
435
454
|
logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
|
436
455
|
|
437
456
|
unless status.success?
|
438
|
-
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
|
457
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
439
458
|
end
|
440
|
-
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}") if out.include?("error")
|
459
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}:#{out}") if out.include?("error")
|
441
460
|
end
|
442
461
|
true
|
443
462
|
end
|
444
463
|
|
445
|
-
def self.delete_public_or_private_keys_v1(email
|
464
|
+
def self.delete_public_or_private_keys_v1(email: nil, key_id: nil, private: false)
|
446
465
|
keys = private ? "secret-keys" : "keys"
|
447
466
|
|
448
|
-
command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email} | grep \"^fpr\" | cut -d: -f10`; do\n"
|
467
|
+
command = "for i in `#{executable} --list-#{keys} --with-colons --fingerprint #{email || key_id} | grep \"^fpr\" | cut -d: -f10`; do\n"
|
449
468
|
command << "#{executable} --batch --no-tty --yes --delete-#{keys} \"$i\" ;\n"
|
450
469
|
command << "done"
|
451
470
|
|
@@ -454,9 +473,9 @@ module IOStreams
|
|
454
473
|
|
455
474
|
return false if err =~ /(not found|no public key)/i
|
456
475
|
unless status.success?
|
457
|
-
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
|
476
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email || key_id}: #{err}: #{out}")
|
458
477
|
end
|
459
|
-
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}") if out.include?("error")
|
478
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email || key_id} #{err.strip}: #{out}") if out.include?("error")
|
460
479
|
|
461
480
|
true
|
462
481
|
end
|