iostreams 0.20.3 → 1.0.0.beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/io_streams/bzip2/reader.rb +9 -21
- data/lib/io_streams/bzip2/writer.rb +9 -21
- data/lib/io_streams/deprecated.rb +217 -0
- data/lib/io_streams/encode/reader.rb +12 -16
- data/lib/io_streams/encode/writer.rb +9 -13
- data/lib/io_streams/errors.rb +6 -6
- data/lib/io_streams/gzip/reader.rb +7 -14
- data/lib/io_streams/gzip/writer.rb +7 -15
- data/lib/io_streams/io_streams.rb +182 -524
- data/lib/io_streams/line/reader.rb +9 -9
- data/lib/io_streams/line/writer.rb +10 -11
- data/lib/io_streams/path.rb +190 -0
- data/lib/io_streams/paths/file.rb +176 -0
- data/lib/io_streams/paths/http.rb +92 -0
- data/lib/io_streams/paths/matcher.rb +61 -0
- data/lib/io_streams/paths/s3.rb +269 -0
- data/lib/io_streams/paths/sftp.rb +99 -0
- data/lib/io_streams/pgp.rb +47 -19
- data/lib/io_streams/pgp/reader.rb +20 -28
- data/lib/io_streams/pgp/writer.rb +24 -46
- data/lib/io_streams/reader.rb +28 -0
- data/lib/io_streams/record/reader.rb +20 -16
- data/lib/io_streams/record/writer.rb +28 -28
- data/lib/io_streams/row/reader.rb +22 -26
- data/lib/io_streams/row/writer.rb +29 -28
- data/lib/io_streams/stream.rb +400 -0
- data/lib/io_streams/streams.rb +125 -0
- data/lib/io_streams/symmetric_encryption/reader.rb +5 -13
- data/lib/io_streams/symmetric_encryption/writer.rb +16 -15
- data/lib/io_streams/tabular/header.rb +9 -3
- data/lib/io_streams/tabular/parser/array.rb +8 -3
- data/lib/io_streams/tabular/parser/csv.rb +6 -2
- data/lib/io_streams/tabular/parser/hash.rb +4 -1
- data/lib/io_streams/tabular/parser/json.rb +3 -1
- data/lib/io_streams/tabular/parser/psv.rb +3 -1
- data/lib/io_streams/tabular/utility/csv_row.rb +9 -8
- data/lib/io_streams/utils.rb +22 -0
- data/lib/io_streams/version.rb +1 -1
- data/lib/io_streams/writer.rb +28 -0
- data/lib/io_streams/xlsx/reader.rb +7 -19
- data/lib/io_streams/zip/reader.rb +7 -26
- data/lib/io_streams/zip/writer.rb +21 -38
- data/lib/iostreams.rb +15 -15
- data/test/bzip2_reader_test.rb +3 -3
- data/test/bzip2_writer_test.rb +3 -3
- data/test/deprecated_test.rb +123 -0
- data/test/encode_reader_test.rb +3 -3
- data/test/encode_writer_test.rb +6 -6
- data/test/gzip_reader_test.rb +2 -2
- data/test/gzip_writer_test.rb +3 -3
- data/test/io_streams_test.rb +43 -136
- data/test/line_reader_test.rb +20 -20
- data/test/line_writer_test.rb +3 -3
- data/test/path_test.rb +30 -28
- data/test/paths/file_test.rb +206 -0
- data/test/paths/http_test.rb +34 -0
- data/test/paths/matcher_test.rb +111 -0
- data/test/paths/s3_test.rb +207 -0
- data/test/pgp_reader_test.rb +8 -8
- data/test/pgp_writer_test.rb +13 -13
- data/test/record_reader_test.rb +5 -5
- data/test/record_writer_test.rb +4 -4
- data/test/row_reader_test.rb +5 -5
- data/test/row_writer_test.rb +6 -6
- data/test/stream_test.rb +116 -0
- data/test/streams_test.rb +255 -0
- data/test/utils_test.rb +20 -0
- data/test/xlsx_reader_test.rb +3 -3
- data/test/zip_reader_test.rb +12 -12
- data/test/zip_writer_test.rb +5 -5
- metadata +33 -45
- data/lib/io_streams/base_path.rb +0 -72
- data/lib/io_streams/file/path.rb +0 -58
- data/lib/io_streams/file/reader.rb +0 -12
- data/lib/io_streams/file/writer.rb +0 -22
- data/lib/io_streams/http/reader.rb +0 -71
- data/lib/io_streams/s3.rb +0 -26
- data/lib/io_streams/s3/path.rb +0 -40
- data/lib/io_streams/s3/reader.rb +0 -28
- data/lib/io_streams/s3/writer.rb +0 -85
- data/lib/io_streams/sftp/reader.rb +0 -67
- data/lib/io_streams/sftp/writer.rb +0 -68
- data/test/base_path_test.rb +0 -35
- data/test/file_path_test.rb +0 -97
- data/test/file_reader_test.rb +0 -33
- data/test/file_writer_test.rb +0 -50
- data/test/http_reader_test.rb +0 -38
- data/test/s3_reader_test.rb +0 -41
- data/test/s3_writer_test.rb +0 -41
@@ -0,0 +1,61 @@
|
|
1
|
+
module IOStreams
|
2
|
+
module Paths
|
3
|
+
# Implement fnmatch logic for any path iterator
|
4
|
+
class Matcher
|
5
|
+
# Characters indicating that pattern matching is required
|
6
|
+
MATCH_START_CHARS = /[*?\[{]/
|
7
|
+
|
8
|
+
attr_reader :path, :pattern, :flags
|
9
|
+
|
10
|
+
# If the supplied pattern contains sub-directories without wildcards, navigate down to that directory
|
11
|
+
# first before applying wildcard lookups from that point on.
|
12
|
+
#
|
13
|
+
# Examples: If the current path is "/path/work"
|
14
|
+
# "a/b/c/**/*" => "/path/work/a/b/c"
|
15
|
+
# "a/b/c?/**/*" => "/path/work/a/b"
|
16
|
+
# "**/*" => "/path/work"
|
17
|
+
#
|
18
|
+
# Note: Absolute paths in the pattern are not supported.
|
19
|
+
def initialize(path, pattern, case_sensitive: false, hidden: false)
|
20
|
+
extract_optimized_path(path, pattern)
|
21
|
+
|
22
|
+
@flags = ::File::FNM_EXTGLOB
|
23
|
+
@flags |= ::File::FNM_CASEFOLD unless case_sensitive
|
24
|
+
@flags |= ::File::FNM_DOTMATCH if hidden
|
25
|
+
end
|
26
|
+
|
27
|
+
# Returns whether the relative `file_name` matches
|
28
|
+
def match?(file_name)
|
29
|
+
relative_file_name = file_name.sub(path.to_s, '').sub(%r{\A/}, '')
|
30
|
+
::File.fnmatch?(pattern, relative_file_name, flags)
|
31
|
+
end
|
32
|
+
|
33
|
+
# Whether this pattern includes a recursive match.
|
34
|
+
# I.e. Includes `**` anywhere in the path
|
35
|
+
def recursive?
|
36
|
+
@recursive ||= pattern.nil? ? false : pattern.include?("**")
|
37
|
+
end
|
38
|
+
|
39
|
+
private
|
40
|
+
|
41
|
+
def extract_optimized_path(path, pattern)
|
42
|
+
elements = pattern.split('/')
|
43
|
+
index = elements.find_index { |e| e.match(MATCH_START_CHARS) }
|
44
|
+
if index == 0
|
45
|
+
# Cannot optimize path since the very first entry contains a wildcard
|
46
|
+
@path = path || IOStreams.path
|
47
|
+
@pattern = pattern
|
48
|
+
elsif index.nil?
|
49
|
+
# No index means it has no pattern.
|
50
|
+
@path = path.nil? ? IOStreams.path(pattern) : path.join(pattern)
|
51
|
+
@pattern = nil
|
52
|
+
else
|
53
|
+
new_path = elements[0..index - 1].join('/')
|
54
|
+
@path = path.nil? ? IOStreams.path(new_path) : path.join(new_path)
|
55
|
+
@pattern = elements[index..-1].join('/')
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,269 @@
|
|
1
|
+
require "uri"
|
2
|
+
|
3
|
+
module IOStreams
|
4
|
+
module Paths
|
5
|
+
class S3 < IOStreams::Path
|
6
|
+
attr_reader :bucket_name, :key, :client
|
7
|
+
|
8
|
+
# Arguments:
|
9
|
+
#
|
10
|
+
# url: [String]
|
11
|
+
# Prefix must be: `s3://`
|
12
|
+
# followed by bucket name,
|
13
|
+
# followed by path and file_name (key).
|
14
|
+
# Examples:
|
15
|
+
# s3://my-bucket-name/file_name.txt
|
16
|
+
# s3://my-bucket-name/some_path/file_name.csv
|
17
|
+
#
|
18
|
+
# Writer specific options:
|
19
|
+
#
|
20
|
+
# @option params [String] :acl
|
21
|
+
# The canned ACL to apply to the object.
|
22
|
+
#
|
23
|
+
# @option params [String] :cache_control
|
24
|
+
# Specifies caching behavior along the request/reply chain.
|
25
|
+
#
|
26
|
+
# @option params [String] :content_disposition
|
27
|
+
# Specifies presentational information for the object.
|
28
|
+
#
|
29
|
+
# @option params [String] :content_encoding
|
30
|
+
# Specifies what content encodings have been applied to the object and
|
31
|
+
# thus what decoding mechanisms must be applied to obtain the media-type
|
32
|
+
# referenced by the Content-Type header field.
|
33
|
+
#
|
34
|
+
# @option params [String] :content_language
|
35
|
+
# The language the content is in.
|
36
|
+
#
|
37
|
+
# @option params [Integer] :content_length
|
38
|
+
# Size of the body in bytes. This parameter is useful when the size of
|
39
|
+
# the body cannot be determined automatically.
|
40
|
+
#
|
41
|
+
# @option params [String] :content_md5
|
42
|
+
# The base64-encoded 128-bit MD5 digest of the part data. This parameter
|
43
|
+
# is auto-populated when using the command from the CLI. This parameted
|
44
|
+
# is required if object lock parameters are specified.
|
45
|
+
#
|
46
|
+
# @option params [String] :content_type
|
47
|
+
# A standard MIME type describing the format of the object data.
|
48
|
+
#
|
49
|
+
# @option params [Time,DateTime,Date,Integer,String] :expires
|
50
|
+
# The date and time at which the object is no longer cacheable.
|
51
|
+
#
|
52
|
+
# @option params [String] :grant_full_control
|
53
|
+
# Gives the grantee READ, READ\_ACP, and WRITE\_ACP permissions on the
|
54
|
+
# object.
|
55
|
+
#
|
56
|
+
# @option params [String] :grant_read
|
57
|
+
# Allows grantee to read the object data and its metadata.
|
58
|
+
#
|
59
|
+
# @option params [String] :grant_read_acp
|
60
|
+
# Allows grantee to read the object ACL.
|
61
|
+
#
|
62
|
+
# @option params [String] :grant_write_acp
|
63
|
+
# Allows grantee to write the ACL for the applicable object.
|
64
|
+
#
|
65
|
+
# @option params [required, String] :key
|
66
|
+
# Object key for which the PUT operation was initiated.
|
67
|
+
#
|
68
|
+
# @option params [Hash<String,String>] :metadata
|
69
|
+
# A map of metadata to store with the object in S3.
|
70
|
+
#
|
71
|
+
# @option params [String] :server_side_encryption
|
72
|
+
# The Server-side encryption algorithm used when storing this object in
|
73
|
+
# S3 (e.g., AES256, aws:kms).
|
74
|
+
#
|
75
|
+
# @option params [String] :storage_class
|
76
|
+
# The type of storage to use for the object. Defaults to 'STANDARD'.
|
77
|
+
#
|
78
|
+
# @option params [String] :website_redirect_location
|
79
|
+
# If the bucket is configured as a website, redirects requests for this
|
80
|
+
# object to another object in the same bucket or to an external URL.
|
81
|
+
# Amazon S3 stores the value of this header in the object metadata.
|
82
|
+
#
|
83
|
+
# @option params [String] :sse_customer_algorithm
|
84
|
+
# Specifies the algorithm to use to when encrypting the object (e.g.,
|
85
|
+
# AES256).
|
86
|
+
#
|
87
|
+
# @option params [String] :sse_customer_key
|
88
|
+
# Specifies the customer-provided encryption key for Amazon S3 to use in
|
89
|
+
# encrypting data. This value is used to store the object and then it is
|
90
|
+
# discarded; Amazon does not store the encryption key. The key must be
|
91
|
+
# appropriate for use with the algorithm specified in the
|
92
|
+
# x-amz-server-side-encryption-customer-algorithm header.
|
93
|
+
#
|
94
|
+
# @option params [String] :sse_customer_key_md5
|
95
|
+
# Specifies the 128-bit MD5 digest of the encryption key according to
|
96
|
+
# RFC 1321. Amazon S3 uses this header for a message integrity check to
|
97
|
+
# ensure the encryption key was transmitted without error.
|
98
|
+
#
|
99
|
+
# @option params [String] :ssekms_key_id
|
100
|
+
# Specifies the AWS KMS key ID to use for object encryption. All GET and
|
101
|
+
# PUT requests for an object protected by AWS KMS will fail if not made
|
102
|
+
# via SSL or using SigV4. Documentation on configuring any of the
|
103
|
+
# officially supported AWS SDKs and CLI can be found at
|
104
|
+
# http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingAWSSDK.html#specify-signature-version
|
105
|
+
#
|
106
|
+
# @option params [String] :ssekms_encryption_context
|
107
|
+
# Specifies the AWS KMS Encryption Context to use for object encryption.
|
108
|
+
# The value of this header is a base64-encoded UTF-8 string holding JSON
|
109
|
+
# with the encryption context key-value pairs.
|
110
|
+
#
|
111
|
+
# @option params [String] :request_payer
|
112
|
+
# Confirms that the requester knows that she or he will be charged for
|
113
|
+
# the request. Bucket owners need not specify this parameter in their
|
114
|
+
# requests. Documentation on downloading objects from requester pays
|
115
|
+
# buckets can be found at
|
116
|
+
# http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
|
117
|
+
#
|
118
|
+
# @option params [String] :tagging
|
119
|
+
# The tag-set for the object. The tag-set must be encoded as URL Query
|
120
|
+
# parameters. (For example, "Key1=Value1")
|
121
|
+
#
|
122
|
+
# @option params [String] :object_lock_mode
|
123
|
+
# The object lock mode that you want to apply to this object.
|
124
|
+
#
|
125
|
+
# @option params [Time,DateTime,Date,Integer,String] :object_lock_retain_until_date
|
126
|
+
# The date and time when you want this object's object lock to expire.
|
127
|
+
#
|
128
|
+
# @option params [String] :object_lock_legal_hold_status
|
129
|
+
# The Legal Hold status that you want to apply to the specified object.
|
130
|
+
def initialize(url, client: nil, **args)
|
131
|
+
Utils.load_dependency('aws-sdk-s3', 'AWS S3') unless defined?(::Aws::S3::Client)
|
132
|
+
|
133
|
+
uri = URI.parse(url)
|
134
|
+
raise "Invalid URI. Required Format: 's3://<bucket_name>/<key>'" unless uri.scheme == 's3'
|
135
|
+
|
136
|
+
@bucket_name = uri.host
|
137
|
+
@key = uri.path.sub(%r{\A/}, '')
|
138
|
+
@client = client || ::Aws::S3::Client.new
|
139
|
+
@options = args
|
140
|
+
super(url)
|
141
|
+
end
|
142
|
+
|
143
|
+
def delete
|
144
|
+
client.delete_object(bucket: bucket_name, key: key)
|
145
|
+
self
|
146
|
+
rescue Aws::S3::Errors::NotFound
|
147
|
+
self
|
148
|
+
end
|
149
|
+
|
150
|
+
def exist?
|
151
|
+
client.head_object(bucket: bucket_name, key: key)
|
152
|
+
true
|
153
|
+
rescue Aws::S3::Errors::NotFound
|
154
|
+
false
|
155
|
+
end
|
156
|
+
|
157
|
+
# Moves this file to the `target_path` by copying it to the new name and then deleting the current file.
|
158
|
+
#
|
159
|
+
# Notes:
|
160
|
+
# - Can copy across buckets.
|
161
|
+
def move_to(target_path)
|
162
|
+
target = IOStreams.new(target_path)
|
163
|
+
return super(target) unless target.is_a?(self.class)
|
164
|
+
|
165
|
+
source_name = ::File.join(bucket_name, key)
|
166
|
+
# TODO: Does/should it also copy metadata?
|
167
|
+
client.copy_object(bucket: target.bucket_name, key: target.key, copy_source: source_name)
|
168
|
+
delete
|
169
|
+
target
|
170
|
+
end
|
171
|
+
|
172
|
+
# S3 logically creates paths when a key is set.
|
173
|
+
def mkpath
|
174
|
+
self
|
175
|
+
end
|
176
|
+
|
177
|
+
def mkdir
|
178
|
+
self
|
179
|
+
end
|
180
|
+
|
181
|
+
def size
|
182
|
+
client.head_object(bucket: bucket_name, key: key).content_length
|
183
|
+
rescue Aws::S3::Errors::NotFound
|
184
|
+
nil
|
185
|
+
end
|
186
|
+
|
187
|
+
# TODO: delete_all
|
188
|
+
|
189
|
+
# Read from AWS S3 file.
|
190
|
+
def reader(&block)
|
191
|
+
# Since S3 download only supports a push stream, write it to a tempfile first.
|
192
|
+
Utils.temp_file_name("iostreams_s3") do |file_name|
|
193
|
+
read_file(file_name)
|
194
|
+
|
195
|
+
::File.open(file_name, 'rb') { |io| io.read }
|
196
|
+
end
|
197
|
+
end
|
198
|
+
|
199
|
+
# Shortcut method if caller has a filename already with no other streams applied:
|
200
|
+
def read_file(file_name)
|
201
|
+
::File.open(file_name, 'wb') do |file|
|
202
|
+
client.get_object(@options.merge(response_target: file, bucket: bucket_name, key: key))
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
# Write to AWS S3
|
207
|
+
#
|
208
|
+
# Raises [MultipartUploadError] If an object is being uploaded in
|
209
|
+
# parts, and the upload can not be completed, then the upload is
|
210
|
+
# aborted and this error is raised. The raised error has a `#errors`
|
211
|
+
# method that returns the failures that caused the upload to be
|
212
|
+
# aborted.
|
213
|
+
def writer(&block)
|
214
|
+
# Since S3 upload only supports a pull stream, write it to a tempfile first.
|
215
|
+
Utils.temp_file_name("iostreams_s3") do |file_name|
|
216
|
+
result = ::File.open(file_name, "wb", &block)
|
217
|
+
|
218
|
+
# Upload file only once all data has been written to it
|
219
|
+
write_file(file_name)
|
220
|
+
result
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
# Shortcut method if caller has a filename already with no other streams applied:
|
225
|
+
def write_file(file_name)
|
226
|
+
if ::File.size(file_name) > 5 * 1024 * 1024
|
227
|
+
# Use multipart file upload
|
228
|
+
s3 = Aws::S3::Resource.new(client: client)
|
229
|
+
obj = s3.bucket(bucket_name).object(key)
|
230
|
+
obj.upload_file(file_name)
|
231
|
+
else
|
232
|
+
::File.open(file_name, 'rb') do |file|
|
233
|
+
client.put_object(@options.merge(bucket: bucket_name, key: key, body: file))
|
234
|
+
end
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
# Notes:
|
239
|
+
# - Currently all S3 lookups are recursive as of the pattern regardless of whether the pattern includes `**`.
|
240
|
+
def each_child(pattern = "*", case_sensitive: false, directories: false, hidden: false)
|
241
|
+
raise(NotImplementedError, "AWS S3 #each_child does not yet return directories") if directories
|
242
|
+
|
243
|
+
matcher = Matcher.new(self, pattern, case_sensitive: case_sensitive, hidden: hidden)
|
244
|
+
|
245
|
+
# When the pattern includes an exact file name without any pattern characters
|
246
|
+
if matcher.pattern.nil?
|
247
|
+
yield(matcher.path) if matcher.path.exist?
|
248
|
+
return
|
249
|
+
end
|
250
|
+
|
251
|
+
prefix = URI.parse(matcher.path.to_s).path.sub(%r{\A/}, '')
|
252
|
+
token = nil
|
253
|
+
loop do
|
254
|
+
# Fetches upto 1,000 entries at a time
|
255
|
+
resp = client.list_objects_v2(bucket: bucket_name, prefix: prefix, continuation_token: token)
|
256
|
+
resp.contents.each do |object|
|
257
|
+
file_name = ::File.join("s3://", resp.name, object.key)
|
258
|
+
next unless matcher.match?(file_name)
|
259
|
+
|
260
|
+
yield self.class.new(file_name)
|
261
|
+
end
|
262
|
+
token = resp.next_continuation_token
|
263
|
+
break if token.nil?
|
264
|
+
end
|
265
|
+
nil
|
266
|
+
end
|
267
|
+
end
|
268
|
+
end
|
269
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
module IOStreams
|
2
|
+
module Paths
|
3
|
+
class SFTP < IOStreams::Path
|
4
|
+
include SemanticLogger::Loggable if defined?(SemanticLogger)
|
5
|
+
|
6
|
+
attr_reader :hostname, :username, :file_name, :create_path, :options
|
7
|
+
|
8
|
+
# Stream to a remote file over sftp.
|
9
|
+
#
|
10
|
+
# file_name: [String]
|
11
|
+
# Name of file to write to.
|
12
|
+
#
|
13
|
+
# username: [String]
|
14
|
+
# Name of user to login with.
|
15
|
+
#
|
16
|
+
# password: [String]
|
17
|
+
# Password for the user.
|
18
|
+
#
|
19
|
+
# host: [String]
|
20
|
+
# Name of the host to connect to.
|
21
|
+
#
|
22
|
+
# port: [Integer]
|
23
|
+
# Port to connect to at the above host.
|
24
|
+
#
|
25
|
+
# **args
|
26
|
+
# Any other options supported by Net::SSH.start
|
27
|
+
#
|
28
|
+
# Examples:
|
29
|
+
#
|
30
|
+
# # Sample URL
|
31
|
+
# sftp://hostname/path/file_name
|
32
|
+
#
|
33
|
+
# # Full url showing all the optional elements that can be set via the url:
|
34
|
+
# sftp://username:password@hostname:22/path/file_name
|
35
|
+
def initialize(url, username:, password:, port: nil, max_pkt_size: 65_536, logger: nil, create_path: false, **args)
|
36
|
+
Utils.load_dependency('net-sftp', 'net/sftp') unless defined?(Net::SFTP)
|
37
|
+
|
38
|
+
uri = URI.parse(url)
|
39
|
+
raise(ArgumentError, "Invalid URL. Required Format: 'sftp://<host_name>/<file_name>'") unless uri.scheme == 'sftp'
|
40
|
+
|
41
|
+
@hostname = uri.hostname
|
42
|
+
@file_name = uri.path
|
43
|
+
@mkdir = false
|
44
|
+
@username = username || uri.user
|
45
|
+
@create_path = create_path
|
46
|
+
|
47
|
+
logger ||= self.logger if defined?(SemanticLogger)
|
48
|
+
options = args.dup
|
49
|
+
options[:logger] = logger
|
50
|
+
options[:port] = port || uri.port || 22
|
51
|
+
options[:max_pkt_size] = max_pkt_size
|
52
|
+
options[:password] = password || uri.password
|
53
|
+
@options = options
|
54
|
+
super(file_name)
|
55
|
+
end
|
56
|
+
|
57
|
+
def mkdir
|
58
|
+
@mkdir = true
|
59
|
+
self
|
60
|
+
end
|
61
|
+
|
62
|
+
# Read a file from a remote sftp server.
|
63
|
+
#
|
64
|
+
# Example:
|
65
|
+
# IOStreams.
|
66
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
67
|
+
# reader do |input|
|
68
|
+
# puts input.read
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# Note:
|
72
|
+
# - raises Net::SFTP::StatusException when the file could not be read.
|
73
|
+
def reader(&block)
|
74
|
+
result = nil
|
75
|
+
Net::SFTP.start(hostname, username, options) do |sftp|
|
76
|
+
result = sftp.file.open(file_name, 'rb', &block)
|
77
|
+
end
|
78
|
+
result
|
79
|
+
end
|
80
|
+
|
81
|
+
# Write to a file on a remote sftp server.
|
82
|
+
#
|
83
|
+
# Example:
|
84
|
+
# IOStreams.
|
85
|
+
# path("sftp://example.org/path/file.txt", username: "jbloggs", password: "secret", compression: false).
|
86
|
+
# writer do |output|
|
87
|
+
# output.write('Hello World')
|
88
|
+
# end
|
89
|
+
def writer(&block)
|
90
|
+
result = nil
|
91
|
+
Net::SFTP.start(hostname, username, options) do |sftp|
|
92
|
+
sftp.session.exec!("mkdir -p '#{::File.dirname(file_name)}'") if create_path
|
93
|
+
result = sftp.file.open(file_name, 'wb', &block)
|
94
|
+
end
|
95
|
+
result
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/io_streams/pgp.rb
CHANGED
@@ -216,7 +216,7 @@ module IOStreams
|
|
216
216
|
# email: [String]
|
217
217
|
def self.key_info(key:)
|
218
218
|
version_check
|
219
|
-
command =
|
219
|
+
command = executable.to_s
|
220
220
|
|
221
221
|
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
222
222
|
logger.debug { "IOStreams::Pgp.key_info: #{command}\n#{err}#{out}" } if logger
|
@@ -275,8 +275,8 @@ module IOStreams
|
|
275
275
|
command = "#{executable} --import"
|
276
276
|
|
277
277
|
out, err, status = Open3.capture3(command, binmode: true, stdin_data: key)
|
278
|
-
logger
|
279
|
-
if status.success? && err.
|
278
|
+
logger&.debug { "IOStreams::Pgp.import: #{command}\n#{err}#{out}" }
|
279
|
+
if status.success? && !err.empty?
|
280
280
|
# Sample output
|
281
281
|
#
|
282
282
|
# gpg: key C16500E3: secret key imported\n"
|
@@ -306,10 +306,26 @@ module IOStreams
|
|
306
306
|
results
|
307
307
|
else
|
308
308
|
return [] if err =~ /already in secret keyring/
|
309
|
+
|
309
310
|
raise(Pgp::Failure, "GPG Failed importing key: #{err}#{out}")
|
310
311
|
end
|
311
312
|
end
|
312
313
|
|
314
|
+
# Returns [String] email for the supplied after importing and trusting the key
|
315
|
+
#
|
316
|
+
# Notes:
|
317
|
+
# - If the same email address has multiple keys then only the first is currently trusted.
|
318
|
+
def self.import_and_trust(key:)
|
319
|
+
raise(ArgumentError, "Key cannot be empty") if key.nil? || (key == '')
|
320
|
+
|
321
|
+
email = key_info(key: key).first.fetch(:email)
|
322
|
+
raise(ArgumentError, "Recipient email cannot be extracted from supplied key") unless email
|
323
|
+
|
324
|
+
import(key: key)
|
325
|
+
set_trust(email: email)
|
326
|
+
email
|
327
|
+
end
|
328
|
+
|
313
329
|
# Set the trust level for an existing key.
|
314
330
|
#
|
315
331
|
# Returns [String] output if the trust was successfully updated
|
@@ -325,7 +341,7 @@ module IOStreams
|
|
325
341
|
command = "#{executable} --import-ownertrust"
|
326
342
|
trust = "#{fingerprint}:#{level + 1}:\n"
|
327
343
|
out, err, status = Open3.capture3(command, stdin_data: trust)
|
328
|
-
logger
|
344
|
+
logger&.debug { "IOStreams::Pgp.set_trust: #{command}\n#{err}#{out}" }
|
329
345
|
if status.success?
|
330
346
|
err
|
331
347
|
else
|
@@ -336,7 +352,7 @@ module IOStreams
|
|
336
352
|
# DEPRECATED - Use key_ids instead of fingerprints
|
337
353
|
def self.fingerprint(email:)
|
338
354
|
version_check
|
339
|
-
Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |
|
355
|
+
Open3.popen2e("#{executable} --list-keys --fingerprint --with-colons #{email}") do |_stdin, out, waith_thr|
|
340
356
|
output = out.read.chomp
|
341
357
|
if waith_thr.value.success?
|
342
358
|
output.each_line do |line|
|
@@ -347,6 +363,7 @@ module IOStreams
|
|
347
363
|
nil
|
348
364
|
else
|
349
365
|
return if output =~ /(public key not found|No public key)/i
|
366
|
+
|
350
367
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email}: #{output}")
|
351
368
|
end
|
352
369
|
end
|
@@ -361,7 +378,7 @@ module IOStreams
|
|
361
378
|
@pgp_version ||= begin
|
362
379
|
command = "#{executable} --version"
|
363
380
|
out, err, status = Open3.capture3(command)
|
364
|
-
logger
|
381
|
+
logger&.debug { "IOStreams::Pgp.version: #{command}\n#{err}#{out}" }
|
365
382
|
if status.success?
|
366
383
|
# Sample output
|
367
384
|
# #{executable} (GnuPG) 2.0.30
|
@@ -383,6 +400,7 @@ module IOStreams
|
|
383
400
|
end
|
384
401
|
else
|
385
402
|
return [] if err =~ /(key not found|No (public|secret) key)/i
|
403
|
+
|
386
404
|
raise(Pgp::Failure, "GPG Failed calling #{executable} to list keys for #{email || key_id}: #{err}#{out}")
|
387
405
|
end
|
388
406
|
end
|
@@ -397,7 +415,9 @@ module IOStreams
|
|
397
415
|
end
|
398
416
|
|
399
417
|
def self.version_check
|
400
|
-
|
418
|
+
if pgp_version.to_f >= 2.3
|
419
|
+
raise(Pgp::UnsupportedVersion, "Version #{pgp_version} of #{executable} is not yet supported. You are welcome to submit a Pull Request.")
|
420
|
+
end
|
401
421
|
end
|
402
422
|
|
403
423
|
# v2.2.1 output:
|
@@ -425,7 +445,7 @@ module IOStreams
|
|
425
445
|
key_type: match[2],
|
426
446
|
date: (Date.parse(match[4].to_s) rescue match[4])
|
427
447
|
}
|
428
|
-
elsif match = line.match(
|
448
|
+
elsif match = line.match(%r{(pub|sec)\s+(\d+)(.*)/(\w+)\s+(\d+-\d+-\d+)(\s+(.+)<(.+)>)?})
|
429
449
|
# Matches: pub 2048R/C7F9D9CB 2016-10-26
|
430
450
|
# Or: pub 2048R/C7F9D9CB 2016-10-26 Receiver <receiver@example.org>
|
431
451
|
hash = {
|
@@ -455,7 +475,6 @@ module IOStreams
|
|
455
475
|
# v2.2 18A0FC1C09C0D8AE34CE659257DC4AE323C7368C
|
456
476
|
hash[:key_id] ||= match[1]
|
457
477
|
end
|
458
|
-
|
459
478
|
end
|
460
479
|
results
|
461
480
|
end
|
@@ -467,13 +486,18 @@ module IOStreams
|
|
467
486
|
return false if list.empty?
|
468
487
|
|
469
488
|
list.each do |key_info|
|
470
|
-
|
471
|
-
|
472
|
-
out, err, status = Open3.capture3(command, binmode: true)
|
473
|
-
logger.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" } if logger
|
489
|
+
key_id = key_info[:key_id]
|
490
|
+
next unless key_id
|
474
491
|
|
475
|
-
|
476
|
-
|
492
|
+
command = "#{executable} --batch --no-tty --yes --delete-#{keys} #{key_id}"
|
493
|
+
out, err, status = Open3.capture3(command, binmode: true)
|
494
|
+
logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}#{out}" }
|
495
|
+
|
496
|
+
unless status.success?
|
497
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
|
498
|
+
end
|
499
|
+
if out.include?('error')
|
500
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}:#{out}")
|
477
501
|
end
|
478
502
|
end
|
479
503
|
true
|
@@ -487,13 +511,17 @@ module IOStreams
|
|
487
511
|
command << 'done'
|
488
512
|
|
489
513
|
out, err, status = Open3.capture3(command, binmode: true)
|
490
|
-
logger
|
514
|
+
logger&.debug { "IOStreams::Pgp.delete_keys: #{command}\n#{err}: #{out}" }
|
491
515
|
|
492
516
|
return false if err =~ /(not found|no public key)/i
|
493
|
-
|
494
|
-
|
517
|
+
unless status.success?
|
518
|
+
raise(Pgp::Failure, "GPG Failed calling #{executable} to delete #{keys} for #{email}: #{err}: #{out}")
|
519
|
+
end
|
520
|
+
if out.include?('error')
|
521
|
+
raise(Pgp::Failure, "GPG Failed to delete #{keys} for #{email} #{err.strip}: #{out}")
|
522
|
+
end
|
523
|
+
|
495
524
|
true
|
496
525
|
end
|
497
|
-
|
498
526
|
end
|
499
527
|
end
|