aws-sdk-s3 1.205.0 → 1.216.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +68 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-s3/bucket.rb +45 -17
- data/lib/aws-sdk-s3/bucket_acl.rb +1 -1
- data/lib/aws-sdk-s3/client.rb +532 -345
- data/lib/aws-sdk-s3/client_api.rb +56 -0
- data/lib/aws-sdk-s3/customizations.rb +9 -2
- data/lib/aws-sdk-s3/directory_download_error.rb +16 -0
- data/lib/aws-sdk-s3/directory_downloader.rb +230 -0
- data/lib/aws-sdk-s3/directory_upload_error.rb +16 -0
- data/lib/aws-sdk-s3/directory_uploader.rb +270 -0
- data/lib/aws-sdk-s3/encryption/client.rb +2 -2
- data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +2 -0
- data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +2 -0
- data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/client.rb +98 -23
- data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +7 -162
- data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
- data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +17 -0
- data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +8 -0
- data/lib/aws-sdk-s3/encryptionV2/utils.rb +5 -0
- data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
- data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
- data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
- data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
- data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
- data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
- data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
- data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
- data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
- data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
- data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
- data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
- data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
- data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
- data/lib/aws-sdk-s3/encryption_v2.rb +1 -0
- data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
- data/lib/aws-sdk-s3/endpoint_provider.rb +21 -18
- data/lib/aws-sdk-s3/endpoints.rb +13 -0
- data/lib/aws-sdk-s3/errors.rb +11 -0
- data/lib/aws-sdk-s3/file_uploader.rb +9 -1
- data/lib/aws-sdk-s3/multipart_file_uploader.rb +4 -1
- data/lib/aws-sdk-s3/multipart_upload.rb +12 -12
- data/lib/aws-sdk-s3/multipart_upload_part.rb +8 -8
- data/lib/aws-sdk-s3/object.rb +36 -36
- data/lib/aws-sdk-s3/object_acl.rb +5 -5
- data/lib/aws-sdk-s3/object_summary.rb +32 -32
- data/lib/aws-sdk-s3/object_version.rb +16 -16
- data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +18 -5
- data/lib/aws-sdk-s3/plugins/http_200_errors.rb +58 -34
- data/lib/aws-sdk-s3/resource.rb +28 -0
- data/lib/aws-sdk-s3/transfer_manager.rb +244 -7
- data/lib/aws-sdk-s3/types.rb +373 -223
- data/lib/aws-sdk-s3.rb +1 -1
- data/sig/bucket.rbs +3 -2
- data/sig/client.rbs +35 -12
- data/sig/errors.rbs +2 -0
- data/sig/multipart_upload.rbs +1 -1
- data/sig/object.rbs +5 -5
- data/sig/object_summary.rbs +5 -5
- data/sig/resource.rbs +2 -1
- data/sig/types.rbs +52 -14
- metadata +23 -3
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'set'
|
|
4
|
+
|
|
5
|
+
module Aws
|
|
6
|
+
module S3
|
|
7
|
+
# @api private
|
|
8
|
+
# This is a one-shot class that uploads files from a local directory to a bucket.
|
|
9
|
+
# This works as follows:
|
|
10
|
+
# * FileProducer runs in a background thread, scanning the directory and
|
|
11
|
+
# pushing entries into a SizedQueue (max: 100).
|
|
12
|
+
# * An internal executor pulls from that queue and posts work. Each task uses
|
|
13
|
+
# FileUploader to upload files then signals completion via `completion_queue`.
|
|
14
|
+
#
|
|
15
|
+
# We track how many tasks we posted, then pop that many times from `completion_queue`
|
|
16
|
+
# to wait for everything to finish.
|
|
17
|
+
#
|
|
18
|
+
# Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
|
|
19
|
+
# we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
|
|
20
|
+
class DirectoryUploader
|
|
21
|
+
def initialize(options = {})
|
|
22
|
+
@client = options[:client] || Client.new
|
|
23
|
+
@executor = options[:executor] || DefaultExecutor.new
|
|
24
|
+
@logger = options[:logger]
|
|
25
|
+
@producer = nil
|
|
26
|
+
@mutex = Mutex.new
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
attr_reader :client, :executor
|
|
30
|
+
|
|
31
|
+
def abort
|
|
32
|
+
@producer&.close
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def upload(source_directory, bucket, **opts)
|
|
36
|
+
raise ArgumentError, 'Invalid directory' unless Dir.exist?(source_directory)
|
|
37
|
+
|
|
38
|
+
uploader = FileUploader.new(
|
|
39
|
+
multipart_threshold: opts.delete(:multipart_threshold),
|
|
40
|
+
http_chunk_size: opts.delete(:http_chunk_size),
|
|
41
|
+
client: @client,
|
|
42
|
+
executor: @executor
|
|
43
|
+
)
|
|
44
|
+
upload_opts = build_upload_opts(opts)
|
|
45
|
+
@producer = FileProducer.new(build_producer_opts(source_directory, bucket, opts))
|
|
46
|
+
uploads, errors = process_upload_queue(uploader, upload_opts)
|
|
47
|
+
build_result(uploads, errors)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
private
|
|
51
|
+
|
|
52
|
+
def build_upload_opts(opts)
|
|
53
|
+
{ ignore_failure: opts[:ignore_failure] || false }
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def build_producer_opts(source_directory, bucket, opts)
|
|
57
|
+
{
|
|
58
|
+
directory_uploader: self,
|
|
59
|
+
source_dir: source_directory,
|
|
60
|
+
bucket: bucket,
|
|
61
|
+
s3_prefix: opts[:s3_prefix],
|
|
62
|
+
recursive: opts[:recursive] || false,
|
|
63
|
+
follow_symlinks: opts[:follow_symlinks] || false,
|
|
64
|
+
filter_callback: opts[:filter_callback],
|
|
65
|
+
request_callback: opts[:request_callback]
|
|
66
|
+
}
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def build_result(upload_count, errors)
|
|
70
|
+
if @producer&.closed?
|
|
71
|
+
msg = "directory upload failed: #{errors.map(&:message).join('; ')}"
|
|
72
|
+
raise DirectoryUploadError.new(msg, errors)
|
|
73
|
+
else
|
|
74
|
+
{
|
|
75
|
+
completed_uploads: [upload_count - errors.count, 0].max,
|
|
76
|
+
failed_uploads: errors.count,
|
|
77
|
+
errors: errors.any? ? errors : nil
|
|
78
|
+
}.compact
|
|
79
|
+
end
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def process_upload_queue(uploader, opts)
|
|
83
|
+
queue_executor = DefaultExecutor.new(max_threads: 2)
|
|
84
|
+
completion_queue = Queue.new
|
|
85
|
+
posted_count = 0
|
|
86
|
+
errors = []
|
|
87
|
+
begin
|
|
88
|
+
@producer.each do |file|
|
|
89
|
+
queue_executor.post(file) do |f|
|
|
90
|
+
upload_file(f, uploader, errors, opts)
|
|
91
|
+
ensure
|
|
92
|
+
completion_queue << :done
|
|
93
|
+
end
|
|
94
|
+
posted_count += 1
|
|
95
|
+
end
|
|
96
|
+
rescue ClosedQueueError
|
|
97
|
+
# abort already requested
|
|
98
|
+
rescue StandardError => e
|
|
99
|
+
@mutex.synchronize { errors << e }
|
|
100
|
+
abort
|
|
101
|
+
end
|
|
102
|
+
posted_count.times { completion_queue.pop }
|
|
103
|
+
[posted_count, errors]
|
|
104
|
+
ensure
|
|
105
|
+
queue_executor&.shutdown
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
def upload_file(entry, uploader, errors, opts)
|
|
109
|
+
uploader.upload(entry.path, entry.params)
|
|
110
|
+
@logger&.debug("Uploaded #{entry.path} to #{entry.params[:bucket]} as #{entry.params[:key]}")
|
|
111
|
+
rescue StandardError => e
|
|
112
|
+
@logger&.warn("Failed to upload #{entry.path} to #{entry.params[:bucket]}: #{e.message}")
|
|
113
|
+
@mutex.synchronize { errors << e }
|
|
114
|
+
abort unless opts[:ignore_failure]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# @api private
|
|
118
|
+
class FileProducer
|
|
119
|
+
include Enumerable
|
|
120
|
+
|
|
121
|
+
DEFAULT_QUEUE_SIZE = 100
|
|
122
|
+
DONE_MARKER = :done
|
|
123
|
+
|
|
124
|
+
def initialize(opts = {})
|
|
125
|
+
@directory_uploader = opts[:directory_uploader]
|
|
126
|
+
@source_dir = opts[:source_dir]
|
|
127
|
+
@bucket = opts[:bucket]
|
|
128
|
+
@s3_prefix = opts[:s3_prefix]
|
|
129
|
+
@recursive = opts[:recursive]
|
|
130
|
+
@follow_symlinks = opts[:follow_symlinks]
|
|
131
|
+
@filter_callback = opts[:filter_callback]
|
|
132
|
+
@request_callback = opts[:request_callback]
|
|
133
|
+
@file_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def closed?
|
|
137
|
+
@file_queue.closed?
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def close
|
|
141
|
+
@file_queue.close
|
|
142
|
+
@file_queue.clear
|
|
143
|
+
end
|
|
144
|
+
|
|
145
|
+
def each
|
|
146
|
+
producer_thread = Thread.new do
|
|
147
|
+
if @recursive
|
|
148
|
+
find_recursively
|
|
149
|
+
else
|
|
150
|
+
find_directly
|
|
151
|
+
end
|
|
152
|
+
@file_queue << DONE_MARKER
|
|
153
|
+
rescue ClosedQueueError
|
|
154
|
+
# abort requested
|
|
155
|
+
rescue StandardError => e
|
|
156
|
+
# encountered a traversal error, we must abort immediately
|
|
157
|
+
close
|
|
158
|
+
raise DirectoryUploadError, "Directory traversal failed for '#{@source_dir}': #{e.message}"
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
while (file = @file_queue.shift) && file != DONE_MARKER
|
|
162
|
+
yield file
|
|
163
|
+
end
|
|
164
|
+
ensure
|
|
165
|
+
producer_thread.value
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
private
|
|
169
|
+
|
|
170
|
+
def apply_request_callback(file_path, params)
|
|
171
|
+
callback_params = @request_callback.call(file_path, params.dup)
|
|
172
|
+
return params unless callback_params.is_a?(Hash) && callback_params.any?
|
|
173
|
+
|
|
174
|
+
params.merge(callback_params)
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def build_upload_entry(file_path, key)
|
|
178
|
+
params = { bucket: @bucket, key: @s3_prefix ? File.join(@s3_prefix, key) : key }
|
|
179
|
+
params = apply_request_callback(file_path, params) if @request_callback
|
|
180
|
+
UploadEntry.new(path: file_path, params: params)
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
def find_directly
|
|
184
|
+
Dir.each_child(@source_dir) do |entry|
|
|
185
|
+
entry_path = File.join(@source_dir, entry)
|
|
186
|
+
stat = nil
|
|
187
|
+
|
|
188
|
+
if @follow_symlinks
|
|
189
|
+
stat = File.stat(entry_path)
|
|
190
|
+
next if stat.directory?
|
|
191
|
+
else
|
|
192
|
+
stat = File.lstat(entry_path)
|
|
193
|
+
next if stat.symlink? || stat.directory?
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
next unless stat.file?
|
|
197
|
+
next unless include_file?(entry_path, entry)
|
|
198
|
+
|
|
199
|
+
@file_queue << build_upload_entry(entry_path, entry)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def find_recursively
|
|
204
|
+
if @follow_symlinks
|
|
205
|
+
ancestors = Set.new
|
|
206
|
+
ancestors << File.stat(@source_dir).ino
|
|
207
|
+
scan_directory(@source_dir, ancestors: ancestors)
|
|
208
|
+
else
|
|
209
|
+
scan_directory(@source_dir)
|
|
210
|
+
end
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def include_file?(file_path, file_name)
|
|
214
|
+
return true unless @filter_callback
|
|
215
|
+
|
|
216
|
+
@filter_callback.call(file_path, file_name)
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def scan_directory(dir_path, key_prefix: '', ancestors: nil)
|
|
220
|
+
Dir.each_child(dir_path) do |entry|
|
|
221
|
+
full_path = File.join(dir_path, entry)
|
|
222
|
+
next unless include_file?(full_path, entry)
|
|
223
|
+
|
|
224
|
+
stat = get_file_stat(full_path)
|
|
225
|
+
next unless stat
|
|
226
|
+
|
|
227
|
+
if stat.directory?
|
|
228
|
+
handle_directory(full_path, entry, key_prefix, ancestors)
|
|
229
|
+
elsif stat.file? # skip non-file types
|
|
230
|
+
key = key_prefix.empty? ? entry : File.join(key_prefix, entry)
|
|
231
|
+
@file_queue << build_upload_entry(full_path, key)
|
|
232
|
+
end
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def get_file_stat(full_path)
|
|
237
|
+
return File.stat(full_path) if @follow_symlinks
|
|
238
|
+
|
|
239
|
+
lstat = File.lstat(full_path)
|
|
240
|
+
return if lstat.symlink?
|
|
241
|
+
|
|
242
|
+
lstat
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
def handle_directory(dir_path, dir_name, key_prefix, ancestors)
|
|
246
|
+
ino = nil
|
|
247
|
+
if @follow_symlinks && ancestors
|
|
248
|
+
ino = File.stat(dir_path).ino
|
|
249
|
+
return if ancestors.include?(ino) # cycle detected - skip
|
|
250
|
+
|
|
251
|
+
ancestors.add(ino)
|
|
252
|
+
end
|
|
253
|
+
new_prefix = key_prefix.empty? ? dir_name : File.join(key_prefix, dir_name)
|
|
254
|
+
scan_directory(dir_path, key_prefix: new_prefix, ancestors: ancestors)
|
|
255
|
+
ancestors.delete(ino) if @follow_symlinks && ancestors
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# @api private
|
|
259
|
+
class UploadEntry
|
|
260
|
+
def initialize(opts = {})
|
|
261
|
+
@path = opts[:path]
|
|
262
|
+
@params = opts[:params]
|
|
263
|
+
end
|
|
264
|
+
|
|
265
|
+
attr_reader :path, :params
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
@@ -6,9 +6,9 @@ module Aws
|
|
|
6
6
|
module S3
|
|
7
7
|
|
|
8
8
|
# [MAINTENANCE MODE] There is a new version of the Encryption Client.
|
|
9
|
-
# AWS strongly recommends upgrading to the {Aws::S3::
|
|
9
|
+
# AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
|
|
10
10
|
# which provides updated data security best practices.
|
|
11
|
-
# See documentation for {Aws::S3::
|
|
11
|
+
# See documentation for {Aws::S3::EncryptionV3::Client}.
|
|
12
12
|
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
13
13
|
# storing the encrypted data in Amazon S3.
|
|
14
14
|
#
|
|
@@ -16,6 +16,8 @@ module Aws
|
|
|
16
16
|
# envelope and encryption cipher.
|
|
17
17
|
def encryption_cipher
|
|
18
18
|
cipher = Utils.aes_encryption_cipher(:CBC)
|
|
19
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
|
|
20
|
+
##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
|
|
19
21
|
envelope = {
|
|
20
22
|
'x-amz-key' => encode64(encrypt(envelope_key(cipher))),
|
|
21
23
|
'x-amz-iv' => encode64(envelope_iv(cipher)),
|
|
@@ -38,6 +38,8 @@ module Aws
|
|
|
38
38
|
io = StringIO.new(io) if String === io
|
|
39
39
|
context.params[:body] = IOEncrypter.new(cipher, io)
|
|
40
40
|
context.params[:metadata] ||= {}
|
|
41
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys
|
|
42
|
+
##% - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects.
|
|
41
43
|
context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
|
|
42
44
|
if context.params.delete(:content_md5)
|
|
43
45
|
warn('Setting content_md5 on client side encrypted objects is deprecated')
|
|
@@ -26,6 +26,8 @@ module Aws
|
|
|
26
26
|
end
|
|
27
27
|
cipher = Utils.aes_encryption_cipher(:CBC)
|
|
28
28
|
cipher.key = key_data.plaintext
|
|
29
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
|
|
30
|
+
##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
|
|
29
31
|
envelope = {
|
|
30
32
|
'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
|
|
31
33
|
'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
|
|
@@ -5,9 +5,17 @@ require 'forwardable'
|
|
|
5
5
|
module Aws
|
|
6
6
|
module S3
|
|
7
7
|
|
|
8
|
-
REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile]
|
|
9
|
-
SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy]
|
|
8
|
+
REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile].freeze
|
|
9
|
+
SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy].freeze
|
|
10
|
+
SUPPORTED_COMMITMENT_POLICIES = [:forbid_encrypt_allow_decrypt].freeze
|
|
10
11
|
|
|
12
|
+
# [MAINTENANCE MODE] There is a new version of the Encryption Client.
|
|
13
|
+
# AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
|
|
14
|
+
# which provides updated data security best practices.
|
|
15
|
+
# For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
|
|
16
|
+
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
17
|
+
# storing the encrypted data in Amazon S3.
|
|
18
|
+
#
|
|
11
19
|
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
12
20
|
# storing the encrypted data in Amazon S3. The `EncryptionV2::Client` (V2 Client)
|
|
13
21
|
# provides improved security over the `Encryption::Client` (V1 Client)
|
|
@@ -307,15 +315,29 @@ module Aws
|
|
|
307
315
|
# @option options [KMS::Client] :kms_client A default {KMS::Client}
|
|
308
316
|
# is constructed when using KMS to manage encryption keys.
|
|
309
317
|
#
|
|
318
|
+
# @option options [Symbol] :commitment_policy (nil)
|
|
319
|
+
# Optional parameter for migration from V2 to V3. When set to
|
|
320
|
+
# :forbid_encrypt_allow_decrypt, this explicitly indicates you are
|
|
321
|
+
# maintaining V2 encryption behavior while preparing for migration.
|
|
322
|
+
# This allows the V2 client to decrypt V3-encrypted objects while
|
|
323
|
+
# continuing to encrypt new objects using V2 algorithms.
|
|
324
|
+
# Only :forbid_encrypt_allow_decrypt is supported.
|
|
325
|
+
# For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
|
|
326
|
+
#
|
|
310
327
|
def initialize(options = {})
|
|
311
328
|
validate_params(options)
|
|
312
329
|
@client = extract_client(options)
|
|
313
|
-
@cipher_provider =
|
|
330
|
+
@cipher_provider = build_cipher_provider(options)
|
|
331
|
+
@key_provider = @cipher_provider.key_provider if @cipher_provider.is_a?(DefaultCipherProvider)
|
|
314
332
|
@envelope_location = extract_location(options)
|
|
315
333
|
@instruction_file_suffix = extract_suffix(options)
|
|
316
334
|
@kms_allow_decrypt_with_any_cmk =
|
|
317
335
|
options[:kms_key_id] == :kms_allow_decrypt_with_any_cmk
|
|
318
336
|
@security_profile = extract_security_profile(options)
|
|
337
|
+
@commitment_policy = extract_commitment_policy(options)
|
|
338
|
+
# The v3 cipher is only used for decrypt.
|
|
339
|
+
# Therefore any configured v2 `content_encryption_schema` is going to be incorrect.
|
|
340
|
+
@v3_cipher_provider = build_v3_cipher_provider_for_decrypt(options.reject { |k, _| k == :content_encryption_schema })
|
|
319
341
|
end
|
|
320
342
|
|
|
321
343
|
# @return [S3::Client]
|
|
@@ -341,6 +363,11 @@ module Aws
|
|
|
341
363
|
# by this string.
|
|
342
364
|
attr_reader :instruction_file_suffix
|
|
343
365
|
|
|
366
|
+
# @return [Symbol, nil] Optional commitment policy for V2 to V3 migration.
|
|
367
|
+
# When set to :forbid_encrypt_allow_decrypt, explicitly indicates
|
|
368
|
+
# maintaining V2 encryption behavior while preparing for migration.
|
|
369
|
+
attr_reader :commitment_policy
|
|
370
|
+
|
|
344
371
|
# Uploads an object to Amazon S3, encrypting data client-side.
|
|
345
372
|
# See {S3::Client#put_object} for documentation on accepted
|
|
346
373
|
# request parameters.
|
|
@@ -410,6 +437,7 @@ module Aws
|
|
|
410
437
|
req.handlers.add(DecryptHandler)
|
|
411
438
|
req.context[:encryption] = {
|
|
412
439
|
cipher_provider: @cipher_provider,
|
|
440
|
+
v3_cipher_provider: @v3_cipher_provider,
|
|
413
441
|
envelope_location: envelope_location,
|
|
414
442
|
instruction_file_suffix: instruction_file_suffix,
|
|
415
443
|
kms_encryption_context: kms_encryption_context,
|
|
@@ -423,6 +451,50 @@ module Aws
|
|
|
423
451
|
|
|
424
452
|
private
|
|
425
453
|
|
|
454
|
+
def build_cipher_provider(options)
|
|
455
|
+
if options[:kms_key_id]
|
|
456
|
+
KmsCipherProvider.new(
|
|
457
|
+
kms_key_id: options[:kms_key_id],
|
|
458
|
+
kms_client: kms_client(options),
|
|
459
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
460
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
461
|
+
)
|
|
462
|
+
else
|
|
463
|
+
key_provider = extract_key_provider(options)
|
|
464
|
+
DefaultCipherProvider.new(
|
|
465
|
+
key_provider: key_provider,
|
|
466
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
467
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
468
|
+
)
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def build_v3_cipher_provider_for_decrypt(options)
|
|
473
|
+
if options[:kms_key_id]
|
|
474
|
+
Aws::S3::EncryptionV3::KmsCipherProvider.new(
|
|
475
|
+
kms_key_id: options[:kms_key_id],
|
|
476
|
+
kms_client: kms_client(options),
|
|
477
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
478
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
479
|
+
)
|
|
480
|
+
else
|
|
481
|
+
# Create V3 key provider explicitly for proper namespace consistency
|
|
482
|
+
key_provider = if options[:key_provider]
|
|
483
|
+
options[:key_provider]
|
|
484
|
+
elsif options[:encryption_key]
|
|
485
|
+
Aws::S3::EncryptionV3::DefaultKeyProvider.new(options)
|
|
486
|
+
else
|
|
487
|
+
msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
|
|
488
|
+
raise ArgumentError, msg
|
|
489
|
+
end
|
|
490
|
+
Aws::S3::EncryptionV3::DefaultCipherProvider.new(
|
|
491
|
+
key_provider: key_provider,
|
|
492
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
493
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
494
|
+
)
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
|
|
426
498
|
# Validate required parameters exist and don't conflict.
|
|
427
499
|
# The cek_alg and wrap_alg are passed on to the CipherProviders
|
|
428
500
|
# and further validated there
|
|
@@ -452,36 +524,19 @@ module Aws
|
|
|
452
524
|
options.delete(:encryption_key)
|
|
453
525
|
options.delete(:envelope_location)
|
|
454
526
|
options.delete(:instruction_file_suffix)
|
|
527
|
+
options.delete(:commitment_policy)
|
|
455
528
|
REQUIRED_PARAMS.each { |p| options.delete(p) }
|
|
456
529
|
S3::Client.new(options)
|
|
457
530
|
end
|
|
458
531
|
end
|
|
459
532
|
|
|
460
533
|
def kms_client(options)
|
|
461
|
-
options[:kms_client] ||
|
|
534
|
+
options[:kms_client] || (@kms_client ||=
|
|
462
535
|
KMS::Client.new(
|
|
463
536
|
region: @client.config.region,
|
|
464
537
|
credentials: @client.config.credentials,
|
|
465
538
|
)
|
|
466
|
-
|
|
467
|
-
end
|
|
468
|
-
|
|
469
|
-
def cipher_provider(options)
|
|
470
|
-
if options[:kms_key_id]
|
|
471
|
-
KmsCipherProvider.new(
|
|
472
|
-
kms_key_id: options[:kms_key_id],
|
|
473
|
-
kms_client: kms_client(options),
|
|
474
|
-
key_wrap_schema: options[:key_wrap_schema],
|
|
475
|
-
content_encryption_schema: options[:content_encryption_schema]
|
|
476
|
-
)
|
|
477
|
-
else
|
|
478
|
-
@key_provider = extract_key_provider(options)
|
|
479
|
-
DefaultCipherProvider.new(
|
|
480
|
-
key_provider: @key_provider,
|
|
481
|
-
key_wrap_schema: options[:key_wrap_schema],
|
|
482
|
-
content_encryption_schema: options[:content_encryption_schema]
|
|
483
|
-
)
|
|
484
|
-
end
|
|
539
|
+
)
|
|
485
540
|
end
|
|
486
541
|
|
|
487
542
|
def extract_key_provider(options)
|
|
@@ -564,7 +619,27 @@ module Aws
|
|
|
564
619
|
end
|
|
565
620
|
security_profile
|
|
566
621
|
end
|
|
622
|
+
|
|
623
|
+
def extract_commitment_policy(options)
|
|
624
|
+
validate_commitment_policy(options[:commitment_policy])
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
def validate_commitment_policy(commitment_policy)
|
|
628
|
+
return nil if commitment_policy.nil?
|
|
629
|
+
|
|
630
|
+
unless SUPPORTED_COMMITMENT_POLICIES.include? commitment_policy
|
|
631
|
+
raise ArgumentError, "Unsupported commitment policy: :#{commitment_policy}. " \
|
|
632
|
+
"The V2 client only supports :forbid_encrypt_allow_decrypt for migration purposes. " \
|
|
633
|
+
"For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html"
|
|
634
|
+
end
|
|
635
|
+
commitment_policy
|
|
636
|
+
end
|
|
567
637
|
end
|
|
568
638
|
end
|
|
569
639
|
end
|
|
570
640
|
end
|
|
641
|
+
|
|
642
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared
|
|
643
|
+
##= type=exception
|
|
644
|
+
##= reason=This has never been supported in Ruby
|
|
645
|
+
##% This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server.
|
|
@@ -9,40 +9,6 @@ module Aws
|
|
|
9
9
|
class DecryptHandler < Seahorse::Client::Handler
|
|
10
10
|
@@warned_response_target_proc = false
|
|
11
11
|
|
|
12
|
-
V1_ENVELOPE_KEYS = %w(
|
|
13
|
-
x-amz-key
|
|
14
|
-
x-amz-iv
|
|
15
|
-
x-amz-matdesc
|
|
16
|
-
)
|
|
17
|
-
|
|
18
|
-
V2_ENVELOPE_KEYS = %w(
|
|
19
|
-
x-amz-key-v2
|
|
20
|
-
x-amz-iv
|
|
21
|
-
x-amz-cek-alg
|
|
22
|
-
x-amz-wrap-alg
|
|
23
|
-
x-amz-matdesc
|
|
24
|
-
)
|
|
25
|
-
|
|
26
|
-
V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
|
|
27
|
-
|
|
28
|
-
POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
|
|
29
|
-
V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
|
|
30
|
-
|
|
31
|
-
POSSIBLE_WRAPPING_FORMATS = %w(
|
|
32
|
-
AES/GCM
|
|
33
|
-
kms
|
|
34
|
-
kms+context
|
|
35
|
-
RSA-OAEP-SHA1
|
|
36
|
-
)
|
|
37
|
-
|
|
38
|
-
POSSIBLE_ENCRYPTION_FORMATS = %w(
|
|
39
|
-
AES/GCM/NoPadding
|
|
40
|
-
AES/CBC/PKCS5Padding
|
|
41
|
-
AES/CBC/PKCS7Padding
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
|
|
45
|
-
|
|
46
12
|
def call(context)
|
|
47
13
|
attach_http_event_listeners(context)
|
|
48
14
|
apply_cse_user_agent(context)
|
|
@@ -63,12 +29,14 @@ module Aws
|
|
|
63
29
|
private
|
|
64
30
|
|
|
65
31
|
def attach_http_event_listeners(context)
|
|
66
|
-
|
|
67
32
|
context.http_response.on_headers(200) do
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
33
|
+
decrypter = if Aws::S3::EncryptionV3::Decryption.v3?(context)
|
|
34
|
+
cipher, envelope = Aws::S3::EncryptionV3::Decryption.decryption_cipher(context)
|
|
35
|
+
Aws::S3::EncryptionV3::Decryption.get_decrypter(context, cipher, envelope)
|
|
36
|
+
else
|
|
37
|
+
cipher, envelope = Aws::S3::EncryptionV2::Decryption.decryption_cipher(context)
|
|
38
|
+
Aws::S3::EncryptionV2::Decryption.get_decrypter(context, cipher, envelope)
|
|
39
|
+
end
|
|
72
40
|
context.http_response.body = decrypter
|
|
73
41
|
end
|
|
74
42
|
|
|
@@ -86,129 +54,6 @@ module Aws
|
|
|
86
54
|
end
|
|
87
55
|
end
|
|
88
56
|
|
|
89
|
-
def decryption_cipher(context)
|
|
90
|
-
if (envelope = get_encryption_envelope(context))
|
|
91
|
-
cipher = context[:encryption][:cipher_provider]
|
|
92
|
-
.decryption_cipher(
|
|
93
|
-
envelope,
|
|
94
|
-
context[:encryption]
|
|
95
|
-
)
|
|
96
|
-
[cipher, envelope]
|
|
97
|
-
else
|
|
98
|
-
raise Errors::DecryptionError, "unable to locate encryption envelope"
|
|
99
|
-
end
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
def get_encryption_envelope(context)
|
|
103
|
-
if context[:encryption][:envelope_location] == :metadata
|
|
104
|
-
envelope_from_metadata(context) || envelope_from_instr_file(context)
|
|
105
|
-
else
|
|
106
|
-
envelope_from_instr_file(context) || envelope_from_metadata(context)
|
|
107
|
-
end
|
|
108
|
-
end
|
|
109
|
-
|
|
110
|
-
def envelope_from_metadata(context)
|
|
111
|
-
possible_envelope = {}
|
|
112
|
-
POSSIBLE_ENVELOPE_KEYS.each do |suffix|
|
|
113
|
-
if value = context.http_response.headers["x-amz-meta-#{suffix}"]
|
|
114
|
-
possible_envelope[suffix] = value
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
extract_envelope(possible_envelope)
|
|
118
|
-
end
|
|
119
|
-
|
|
120
|
-
def envelope_from_instr_file(context)
|
|
121
|
-
suffix = context[:encryption][:instruction_file_suffix]
|
|
122
|
-
possible_envelope = Json.load(context.client.get_object(
|
|
123
|
-
bucket: context.params[:bucket],
|
|
124
|
-
key: context.params[:key] + suffix
|
|
125
|
-
).body.read)
|
|
126
|
-
extract_envelope(possible_envelope)
|
|
127
|
-
rescue S3::Errors::ServiceError, Json::ParseError
|
|
128
|
-
nil
|
|
129
|
-
end
|
|
130
|
-
|
|
131
|
-
def extract_envelope(hash)
|
|
132
|
-
return nil unless hash
|
|
133
|
-
return v1_envelope(hash) if hash.key?('x-amz-key')
|
|
134
|
-
return v2_envelope(hash) if hash.key?('x-amz-key-v2')
|
|
135
|
-
if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
|
|
136
|
-
msg = "unsupported envelope encryption version #{$1}"
|
|
137
|
-
raise Errors::DecryptionError, msg
|
|
138
|
-
end
|
|
139
|
-
end
|
|
140
|
-
|
|
141
|
-
def v1_envelope(envelope)
|
|
142
|
-
envelope
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
def v2_envelope(envelope)
|
|
146
|
-
unless POSSIBLE_ENCRYPTION_FORMATS.include? envelope['x-amz-cek-alg']
|
|
147
|
-
alg = envelope['x-amz-cek-alg'].inspect
|
|
148
|
-
msg = "unsupported content encrypting key (cek) format: #{alg}"
|
|
149
|
-
raise Errors::DecryptionError, msg
|
|
150
|
-
end
|
|
151
|
-
unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
|
|
152
|
-
alg = envelope['x-amz-wrap-alg'].inspect
|
|
153
|
-
msg = "unsupported key wrapping algorithm: #{alg}"
|
|
154
|
-
raise Errors::DecryptionError, msg
|
|
155
|
-
end
|
|
156
|
-
unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
|
|
157
|
-
msg = "incomplete v2 encryption envelope:\n"
|
|
158
|
-
msg += " missing: #{missing_keys.join(',')}\n"
|
|
159
|
-
raise Errors::DecryptionError, msg
|
|
160
|
-
end
|
|
161
|
-
envelope
|
|
162
|
-
end
|
|
163
|
-
|
|
164
|
-
# This method fetches the tag from the end of the object by
|
|
165
|
-
# making a GET Object w/range request. This auth tag is used
|
|
166
|
-
# to initialize the cipher, and the decrypter truncates the
|
|
167
|
-
# auth tag from the body when writing the final bytes.
|
|
168
|
-
def authenticated_decrypter(context, cipher, envelope)
|
|
169
|
-
http_resp = context.http_response
|
|
170
|
-
content_length = http_resp.headers['content-length'].to_i
|
|
171
|
-
auth_tag_length = auth_tag_length(envelope)
|
|
172
|
-
|
|
173
|
-
auth_tag = context.client.get_object(
|
|
174
|
-
bucket: context.params[:bucket],
|
|
175
|
-
key: context.params[:key],
|
|
176
|
-
version_id: context.params[:version_id],
|
|
177
|
-
range: "bytes=-#{auth_tag_length}"
|
|
178
|
-
).body.read
|
|
179
|
-
|
|
180
|
-
cipher.auth_tag = auth_tag
|
|
181
|
-
cipher.auth_data = ''
|
|
182
|
-
|
|
183
|
-
# The encrypted object contains both the cipher text
|
|
184
|
-
# plus a trailing auth tag.
|
|
185
|
-
IOAuthDecrypter.new(
|
|
186
|
-
io: http_resp.body,
|
|
187
|
-
encrypted_content_length: content_length - auth_tag_length,
|
|
188
|
-
cipher: cipher)
|
|
189
|
-
end
|
|
190
|
-
|
|
191
|
-
def body_contains_auth_tag?(envelope)
|
|
192
|
-
AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
|
|
193
|
-
end
|
|
194
|
-
|
|
195
|
-
# Determine the auth tag length from the algorithm
|
|
196
|
-
# Validate it against the value provided in the x-amz-tag-len
|
|
197
|
-
# Return the tag length in bytes
|
|
198
|
-
def auth_tag_length(envelope)
|
|
199
|
-
tag_length =
|
|
200
|
-
case envelope['x-amz-cek-alg']
|
|
201
|
-
when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
|
|
202
|
-
else
|
|
203
|
-
raise ArgumentError, 'Unsupported cek-alg: ' \
|
|
204
|
-
"#{envelope['x-amz-cek-alg']}"
|
|
205
|
-
end
|
|
206
|
-
if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
|
|
207
|
-
raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
|
|
208
|
-
end
|
|
209
|
-
tag_length
|
|
210
|
-
end
|
|
211
|
-
|
|
212
57
|
def apply_cse_user_agent(context)
|
|
213
58
|
if context.config.user_agent_suffix.nil?
|
|
214
59
|
context.config.user_agent_suffix = EC_USER_AGENT
|