aws-sdk-s3 1.205.0 → 1.216.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +68 -0
  3. data/VERSION +1 -1
  4. data/lib/aws-sdk-s3/bucket.rb +45 -17
  5. data/lib/aws-sdk-s3/bucket_acl.rb +1 -1
  6. data/lib/aws-sdk-s3/client.rb +532 -345
  7. data/lib/aws-sdk-s3/client_api.rb +56 -0
  8. data/lib/aws-sdk-s3/customizations.rb +9 -2
  9. data/lib/aws-sdk-s3/directory_download_error.rb +16 -0
  10. data/lib/aws-sdk-s3/directory_downloader.rb +230 -0
  11. data/lib/aws-sdk-s3/directory_upload_error.rb +16 -0
  12. data/lib/aws-sdk-s3/directory_uploader.rb +270 -0
  13. data/lib/aws-sdk-s3/encryption/client.rb +2 -2
  14. data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +2 -0
  15. data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +2 -0
  16. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +2 -0
  17. data/lib/aws-sdk-s3/encryptionV2/client.rb +98 -23
  18. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +7 -162
  19. data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
  20. data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +17 -0
  21. data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +2 -0
  22. data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +2 -0
  23. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +8 -0
  24. data/lib/aws-sdk-s3/encryptionV2/utils.rb +5 -0
  25. data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
  26. data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
  27. data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
  28. data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
  29. data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
  30. data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
  31. data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
  32. data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
  33. data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
  34. data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
  35. data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
  36. data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
  37. data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
  38. data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
  39. data/lib/aws-sdk-s3/encryption_v2.rb +1 -0
  40. data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
  41. data/lib/aws-sdk-s3/endpoint_provider.rb +21 -18
  42. data/lib/aws-sdk-s3/endpoints.rb +13 -0
  43. data/lib/aws-sdk-s3/errors.rb +11 -0
  44. data/lib/aws-sdk-s3/file_uploader.rb +9 -1
  45. data/lib/aws-sdk-s3/multipart_file_uploader.rb +4 -1
  46. data/lib/aws-sdk-s3/multipart_upload.rb +12 -12
  47. data/lib/aws-sdk-s3/multipart_upload_part.rb +8 -8
  48. data/lib/aws-sdk-s3/object.rb +36 -36
  49. data/lib/aws-sdk-s3/object_acl.rb +5 -5
  50. data/lib/aws-sdk-s3/object_summary.rb +32 -32
  51. data/lib/aws-sdk-s3/object_version.rb +16 -16
  52. data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +18 -5
  53. data/lib/aws-sdk-s3/plugins/http_200_errors.rb +58 -34
  54. data/lib/aws-sdk-s3/resource.rb +28 -0
  55. data/lib/aws-sdk-s3/transfer_manager.rb +244 -7
  56. data/lib/aws-sdk-s3/types.rb +373 -223
  57. data/lib/aws-sdk-s3.rb +1 -1
  58. data/sig/bucket.rbs +3 -2
  59. data/sig/client.rbs +35 -12
  60. data/sig/errors.rbs +2 -0
  61. data/sig/multipart_upload.rbs +1 -1
  62. data/sig/object.rbs +5 -5
  63. data/sig/object_summary.rbs +5 -5
  64. data/sig/resource.rbs +2 -1
  65. data/sig/types.rbs +52 -14
  66. metadata +23 -3
@@ -0,0 +1,270 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+
5
+ module Aws
6
+ module S3
7
+ # @api private
8
+ # This is a one-shot class that uploads files from a local directory to a bucket.
9
+ # This works as follows:
10
+ # * FileProducer runs in a background thread, scanning the directory and
11
+ # pushing entries into a SizedQueue (max: 100).
12
+ # * An internal executor pulls from that queue and posts work. Each task uses
13
+ # FileUploader to upload files then signals completion via `completion_queue`.
14
+ #
15
+ # We track how many tasks we posted, then pop that many times from `completion_queue`
16
+ # to wait for everything to finish.
17
+ #
18
+ # Errors are collected in a mutex-protected array. On failure (unless ignore_failure is set),
19
+ # we call abort which closes the queue - the producer catches ClosedQueueError and exits cleanly.
20
+ class DirectoryUploader
21
+ def initialize(options = {})
22
+ @client = options[:client] || Client.new
23
+ @executor = options[:executor] || DefaultExecutor.new
24
+ @logger = options[:logger]
25
+ @producer = nil
26
+ @mutex = Mutex.new
27
+ end
28
+
29
+ attr_reader :client, :executor
30
+
31
+ def abort
32
+ @producer&.close
33
+ end
34
+
35
+ def upload(source_directory, bucket, **opts)
36
+ raise ArgumentError, 'Invalid directory' unless Dir.exist?(source_directory)
37
+
38
+ uploader = FileUploader.new(
39
+ multipart_threshold: opts.delete(:multipart_threshold),
40
+ http_chunk_size: opts.delete(:http_chunk_size),
41
+ client: @client,
42
+ executor: @executor
43
+ )
44
+ upload_opts = build_upload_opts(opts)
45
+ @producer = FileProducer.new(build_producer_opts(source_directory, bucket, opts))
46
+ uploads, errors = process_upload_queue(uploader, upload_opts)
47
+ build_result(uploads, errors)
48
+ end
49
+
50
+ private
51
+
52
+ def build_upload_opts(opts)
53
+ { ignore_failure: opts[:ignore_failure] || false }
54
+ end
55
+
56
+ def build_producer_opts(source_directory, bucket, opts)
57
+ {
58
+ directory_uploader: self,
59
+ source_dir: source_directory,
60
+ bucket: bucket,
61
+ s3_prefix: opts[:s3_prefix],
62
+ recursive: opts[:recursive] || false,
63
+ follow_symlinks: opts[:follow_symlinks] || false,
64
+ filter_callback: opts[:filter_callback],
65
+ request_callback: opts[:request_callback]
66
+ }
67
+ end
68
+
69
+ def build_result(upload_count, errors)
70
+ if @producer&.closed?
71
+ msg = "directory upload failed: #{errors.map(&:message).join('; ')}"
72
+ raise DirectoryUploadError.new(msg, errors)
73
+ else
74
+ {
75
+ completed_uploads: [upload_count - errors.count, 0].max,
76
+ failed_uploads: errors.count,
77
+ errors: errors.any? ? errors : nil
78
+ }.compact
79
+ end
80
+ end
81
+
82
+ def process_upload_queue(uploader, opts)
83
+ queue_executor = DefaultExecutor.new(max_threads: 2)
84
+ completion_queue = Queue.new
85
+ posted_count = 0
86
+ errors = []
87
+ begin
88
+ @producer.each do |file|
89
+ queue_executor.post(file) do |f|
90
+ upload_file(f, uploader, errors, opts)
91
+ ensure
92
+ completion_queue << :done
93
+ end
94
+ posted_count += 1
95
+ end
96
+ rescue ClosedQueueError
97
+ # abort already requested
98
+ rescue StandardError => e
99
+ @mutex.synchronize { errors << e }
100
+ abort
101
+ end
102
+ posted_count.times { completion_queue.pop }
103
+ [posted_count, errors]
104
+ ensure
105
+ queue_executor&.shutdown
106
+ end
107
+
108
+ def upload_file(entry, uploader, errors, opts)
109
+ uploader.upload(entry.path, entry.params)
110
+ @logger&.debug("Uploaded #{entry.path} to #{entry.params[:bucket]} as #{entry.params[:key]}")
111
+ rescue StandardError => e
112
+ @logger&.warn("Failed to upload #{entry.path} to #{entry.params[:bucket]}: #{e.message}")
113
+ @mutex.synchronize { errors << e }
114
+ abort unless opts[:ignore_failure]
115
+ end
116
+
117
+ # @api private
118
+ class FileProducer
119
+ include Enumerable
120
+
121
+ DEFAULT_QUEUE_SIZE = 100
122
+ DONE_MARKER = :done
123
+
124
+ def initialize(opts = {})
125
+ @directory_uploader = opts[:directory_uploader]
126
+ @source_dir = opts[:source_dir]
127
+ @bucket = opts[:bucket]
128
+ @s3_prefix = opts[:s3_prefix]
129
+ @recursive = opts[:recursive]
130
+ @follow_symlinks = opts[:follow_symlinks]
131
+ @filter_callback = opts[:filter_callback]
132
+ @request_callback = opts[:request_callback]
133
+ @file_queue = SizedQueue.new(DEFAULT_QUEUE_SIZE)
134
+ end
135
+
136
+ def closed?
137
+ @file_queue.closed?
138
+ end
139
+
140
+ def close
141
+ @file_queue.close
142
+ @file_queue.clear
143
+ end
144
+
145
+ def each
146
+ producer_thread = Thread.new do
147
+ if @recursive
148
+ find_recursively
149
+ else
150
+ find_directly
151
+ end
152
+ @file_queue << DONE_MARKER
153
+ rescue ClosedQueueError
154
+ # abort requested
155
+ rescue StandardError => e
156
+ # encountered a traversal error, we must abort immediately
157
+ close
158
+ raise DirectoryUploadError, "Directory traversal failed for '#{@source_dir}': #{e.message}"
159
+ end
160
+
161
+ while (file = @file_queue.shift) && file != DONE_MARKER
162
+ yield file
163
+ end
164
+ ensure
165
+ producer_thread.value
166
+ end
167
+
168
+ private
169
+
170
+ def apply_request_callback(file_path, params)
171
+ callback_params = @request_callback.call(file_path, params.dup)
172
+ return params unless callback_params.is_a?(Hash) && callback_params.any?
173
+
174
+ params.merge(callback_params)
175
+ end
176
+
177
+ def build_upload_entry(file_path, key)
178
+ params = { bucket: @bucket, key: @s3_prefix ? File.join(@s3_prefix, key) : key }
179
+ params = apply_request_callback(file_path, params) if @request_callback
180
+ UploadEntry.new(path: file_path, params: params)
181
+ end
182
+
183
+ def find_directly
184
+ Dir.each_child(@source_dir) do |entry|
185
+ entry_path = File.join(@source_dir, entry)
186
+ stat = nil
187
+
188
+ if @follow_symlinks
189
+ stat = File.stat(entry_path)
190
+ next if stat.directory?
191
+ else
192
+ stat = File.lstat(entry_path)
193
+ next if stat.symlink? || stat.directory?
194
+ end
195
+
196
+ next unless stat.file?
197
+ next unless include_file?(entry_path, entry)
198
+
199
+ @file_queue << build_upload_entry(entry_path, entry)
200
+ end
201
+ end
202
+
203
+ def find_recursively
204
+ if @follow_symlinks
205
+ ancestors = Set.new
206
+ ancestors << File.stat(@source_dir).ino
207
+ scan_directory(@source_dir, ancestors: ancestors)
208
+ else
209
+ scan_directory(@source_dir)
210
+ end
211
+ end
212
+
213
+ def include_file?(file_path, file_name)
214
+ return true unless @filter_callback
215
+
216
+ @filter_callback.call(file_path, file_name)
217
+ end
218
+
219
+ def scan_directory(dir_path, key_prefix: '', ancestors: nil)
220
+ Dir.each_child(dir_path) do |entry|
221
+ full_path = File.join(dir_path, entry)
222
+ next unless include_file?(full_path, entry)
223
+
224
+ stat = get_file_stat(full_path)
225
+ next unless stat
226
+
227
+ if stat.directory?
228
+ handle_directory(full_path, entry, key_prefix, ancestors)
229
+ elsif stat.file? # skip non-file types
230
+ key = key_prefix.empty? ? entry : File.join(key_prefix, entry)
231
+ @file_queue << build_upload_entry(full_path, key)
232
+ end
233
+ end
234
+ end
235
+
236
+ def get_file_stat(full_path)
237
+ return File.stat(full_path) if @follow_symlinks
238
+
239
+ lstat = File.lstat(full_path)
240
+ return if lstat.symlink?
241
+
242
+ lstat
243
+ end
244
+
245
+ def handle_directory(dir_path, dir_name, key_prefix, ancestors)
246
+ ino = nil
247
+ if @follow_symlinks && ancestors
248
+ ino = File.stat(dir_path).ino
249
+ return if ancestors.include?(ino) # cycle detected - skip
250
+
251
+ ancestors.add(ino)
252
+ end
253
+ new_prefix = key_prefix.empty? ? dir_name : File.join(key_prefix, dir_name)
254
+ scan_directory(dir_path, key_prefix: new_prefix, ancestors: ancestors)
255
+ ancestors.delete(ino) if @follow_symlinks && ancestors
256
+ end
257
+
258
+ # @api private
259
+ class UploadEntry
260
+ def initialize(opts = {})
261
+ @path = opts[:path]
262
+ @params = opts[:params]
263
+ end
264
+
265
+ attr_reader :path, :params
266
+ end
267
+ end
268
+ end
269
+ end
270
+ end
@@ -6,9 +6,9 @@ module Aws
6
6
  module S3
7
7
 
8
8
  # [MAINTENANCE MODE] There is a new version of the Encryption Client.
9
- # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV2::Client},
9
+ # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
10
10
  # which provides updated data security best practices.
11
- # See documentation for {Aws::S3::EncryptionV2::Client}.
11
+ # See documentation for {Aws::S3::EncryptionV3::Client}.
12
12
  # Provides an encryption client that encrypts and decrypts data client-side,
13
13
  # storing the encrypted data in Amazon S3.
14
14
  #
@@ -16,6 +16,8 @@ module Aws
16
16
  # envelope and encryption cipher.
17
17
  def encryption_cipher
18
18
  cipher = Utils.aes_encryption_cipher(:CBC)
19
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
20
+ ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
19
21
  envelope = {
20
22
  'x-amz-key' => encode64(encrypt(envelope_key(cipher))),
21
23
  'x-amz-iv' => encode64(envelope_iv(cipher)),
@@ -38,6 +38,8 @@ module Aws
38
38
  io = StringIO.new(io) if String === io
39
39
  context.params[:body] = IOEncrypter.new(cipher, io)
40
40
  context.params[:metadata] ||= {}
41
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys
42
+ ##% - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects.
41
43
  context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
42
44
  if context.params.delete(:content_md5)
43
45
  warn('Setting content_md5 on client side encrypted objects is deprecated')
@@ -26,6 +26,8 @@ module Aws
26
26
  end
27
27
  cipher = Utils.aes_encryption_cipher(:CBC)
28
28
  cipher.key = key_data.plaintext
29
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
30
+ ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
29
31
  envelope = {
30
32
  'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
31
33
  'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
@@ -5,9 +5,17 @@ require 'forwardable'
5
5
  module Aws
6
6
  module S3
7
7
 
8
- REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile]
9
- SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy]
8
+ REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile].freeze
9
+ SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy].freeze
10
+ SUPPORTED_COMMITMENT_POLICIES = [:forbid_encrypt_allow_decrypt].freeze
10
11
 
12
+ # [MAINTENANCE MODE] There is a new version of the Encryption Client.
13
+ # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
14
+ # which provides updated data security best practices.
15
+ # For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
16
+ # Provides an encryption client that encrypts and decrypts data client-side,
17
+ # storing the encrypted data in Amazon S3.
18
+ #
11
19
  # Provides an encryption client that encrypts and decrypts data client-side,
12
20
  # storing the encrypted data in Amazon S3. The `EncryptionV2::Client` (V2 Client)
13
21
  # provides improved security over the `Encryption::Client` (V1 Client)
@@ -307,15 +315,29 @@ module Aws
307
315
  # @option options [KMS::Client] :kms_client A default {KMS::Client}
308
316
  # is constructed when using KMS to manage encryption keys.
309
317
  #
318
+ # @option options [Symbol] :commitment_policy (nil)
319
+ # Optional parameter for migration from V2 to V3. When set to
320
+ # :forbid_encrypt_allow_decrypt, this explicitly indicates you are
321
+ # maintaining V2 encryption behavior while preparing for migration.
322
+ # This allows the V2 client to decrypt V3-encrypted objects while
323
+ # continuing to encrypt new objects using V2 algorithms.
324
+ # Only :forbid_encrypt_allow_decrypt is supported.
325
+ # For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
326
+ #
310
327
  def initialize(options = {})
311
328
  validate_params(options)
312
329
  @client = extract_client(options)
313
- @cipher_provider = cipher_provider(options)
330
+ @cipher_provider = build_cipher_provider(options)
331
+ @key_provider = @cipher_provider.key_provider if @cipher_provider.is_a?(DefaultCipherProvider)
314
332
  @envelope_location = extract_location(options)
315
333
  @instruction_file_suffix = extract_suffix(options)
316
334
  @kms_allow_decrypt_with_any_cmk =
317
335
  options[:kms_key_id] == :kms_allow_decrypt_with_any_cmk
318
336
  @security_profile = extract_security_profile(options)
337
+ @commitment_policy = extract_commitment_policy(options)
338
+ # The v3 cipher is only used for decrypt.
339
+ # Therefore any configured v2 `content_encryption_schema` is going to be incorrect.
340
+ @v3_cipher_provider = build_v3_cipher_provider_for_decrypt(options.reject { |k, _| k == :content_encryption_schema })
319
341
  end
320
342
 
321
343
  # @return [S3::Client]
@@ -341,6 +363,11 @@ module Aws
341
363
  # by this string.
342
364
  attr_reader :instruction_file_suffix
343
365
 
366
+ # @return [Symbol, nil] Optional commitment policy for V2 to V3 migration.
367
+ # When set to :forbid_encrypt_allow_decrypt, explicitly indicates
368
+ # maintaining V2 encryption behavior while preparing for migration.
369
+ attr_reader :commitment_policy
370
+
344
371
  # Uploads an object to Amazon S3, encrypting data client-side.
345
372
  # See {S3::Client#put_object} for documentation on accepted
346
373
  # request parameters.
@@ -410,6 +437,7 @@ module Aws
410
437
  req.handlers.add(DecryptHandler)
411
438
  req.context[:encryption] = {
412
439
  cipher_provider: @cipher_provider,
440
+ v3_cipher_provider: @v3_cipher_provider,
413
441
  envelope_location: envelope_location,
414
442
  instruction_file_suffix: instruction_file_suffix,
415
443
  kms_encryption_context: kms_encryption_context,
@@ -423,6 +451,50 @@ module Aws
423
451
 
424
452
  private
425
453
 
454
+ def build_cipher_provider(options)
455
+ if options[:kms_key_id]
456
+ KmsCipherProvider.new(
457
+ kms_key_id: options[:kms_key_id],
458
+ kms_client: kms_client(options),
459
+ key_wrap_schema: options[:key_wrap_schema],
460
+ content_encryption_schema: options[:content_encryption_schema]
461
+ )
462
+ else
463
+ key_provider = extract_key_provider(options)
464
+ DefaultCipherProvider.new(
465
+ key_provider: key_provider,
466
+ key_wrap_schema: options[:key_wrap_schema],
467
+ content_encryption_schema: options[:content_encryption_schema]
468
+ )
469
+ end
470
+ end
471
+
472
+ def build_v3_cipher_provider_for_decrypt(options)
473
+ if options[:kms_key_id]
474
+ Aws::S3::EncryptionV3::KmsCipherProvider.new(
475
+ kms_key_id: options[:kms_key_id],
476
+ kms_client: kms_client(options),
477
+ key_wrap_schema: options[:key_wrap_schema],
478
+ content_encryption_schema: options[:content_encryption_schema]
479
+ )
480
+ else
481
+ # Create V3 key provider explicitly for proper namespace consistency
482
+ key_provider = if options[:key_provider]
483
+ options[:key_provider]
484
+ elsif options[:encryption_key]
485
+ Aws::S3::EncryptionV3::DefaultKeyProvider.new(options)
486
+ else
487
+ msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
488
+ raise ArgumentError, msg
489
+ end
490
+ Aws::S3::EncryptionV3::DefaultCipherProvider.new(
491
+ key_provider: key_provider,
492
+ key_wrap_schema: options[:key_wrap_schema],
493
+ content_encryption_schema: options[:content_encryption_schema]
494
+ )
495
+ end
496
+ end
497
+
426
498
  # Validate required parameters exist and don't conflict.
427
499
  # The cek_alg and wrap_alg are passed on to the CipherProviders
428
500
  # and further validated there
@@ -452,36 +524,19 @@ module Aws
452
524
  options.delete(:encryption_key)
453
525
  options.delete(:envelope_location)
454
526
  options.delete(:instruction_file_suffix)
527
+ options.delete(:commitment_policy)
455
528
  REQUIRED_PARAMS.each { |p| options.delete(p) }
456
529
  S3::Client.new(options)
457
530
  end
458
531
  end
459
532
 
460
533
  def kms_client(options)
461
- options[:kms_client] || begin
534
+ options[:kms_client] || (@kms_client ||=
462
535
  KMS::Client.new(
463
536
  region: @client.config.region,
464
537
  credentials: @client.config.credentials,
465
538
  )
466
- end
467
- end
468
-
469
- def cipher_provider(options)
470
- if options[:kms_key_id]
471
- KmsCipherProvider.new(
472
- kms_key_id: options[:kms_key_id],
473
- kms_client: kms_client(options),
474
- key_wrap_schema: options[:key_wrap_schema],
475
- content_encryption_schema: options[:content_encryption_schema]
476
- )
477
- else
478
- @key_provider = extract_key_provider(options)
479
- DefaultCipherProvider.new(
480
- key_provider: @key_provider,
481
- key_wrap_schema: options[:key_wrap_schema],
482
- content_encryption_schema: options[:content_encryption_schema]
483
- )
484
- end
539
+ )
485
540
  end
486
541
 
487
542
  def extract_key_provider(options)
@@ -564,7 +619,27 @@ module Aws
564
619
  end
565
620
  security_profile
566
621
  end
622
+
623
+ def extract_commitment_policy(options)
624
+ validate_commitment_policy(options[:commitment_policy])
625
+ end
626
+
627
+ def validate_commitment_policy(commitment_policy)
628
+ return nil if commitment_policy.nil?
629
+
630
+ unless SUPPORTED_COMMITMENT_POLICIES.include? commitment_policy
631
+ raise ArgumentError, "Unsupported commitment policy: :#{commitment_policy}. " \
632
+ "The V2 client only supports :forbid_encrypt_allow_decrypt for migration purposes. " \
633
+ "For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html"
634
+ end
635
+ commitment_policy
636
+ end
567
637
  end
568
638
  end
569
639
  end
570
640
  end
641
+
642
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared
643
+ ##= type=exception
644
+ ##= reason=This has never been supported in Ruby
645
+ ##% This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server.
@@ -9,40 +9,6 @@ module Aws
9
9
  class DecryptHandler < Seahorse::Client::Handler
10
10
  @@warned_response_target_proc = false
11
11
 
12
- V1_ENVELOPE_KEYS = %w(
13
- x-amz-key
14
- x-amz-iv
15
- x-amz-matdesc
16
- )
17
-
18
- V2_ENVELOPE_KEYS = %w(
19
- x-amz-key-v2
20
- x-amz-iv
21
- x-amz-cek-alg
22
- x-amz-wrap-alg
23
- x-amz-matdesc
24
- )
25
-
26
- V2_OPTIONAL_KEYS = %w(x-amz-tag-len)
27
-
28
- POSSIBLE_ENVELOPE_KEYS = (V1_ENVELOPE_KEYS +
29
- V2_ENVELOPE_KEYS + V2_OPTIONAL_KEYS).uniq
30
-
31
- POSSIBLE_WRAPPING_FORMATS = %w(
32
- AES/GCM
33
- kms
34
- kms+context
35
- RSA-OAEP-SHA1
36
- )
37
-
38
- POSSIBLE_ENCRYPTION_FORMATS = %w(
39
- AES/GCM/NoPadding
40
- AES/CBC/PKCS5Padding
41
- AES/CBC/PKCS7Padding
42
- )
43
-
44
- AUTH_REQUIRED_CEK_ALGS = %w(AES/GCM/NoPadding)
45
-
46
12
  def call(context)
47
13
  attach_http_event_listeners(context)
48
14
  apply_cse_user_agent(context)
@@ -63,12 +29,14 @@ module Aws
63
29
  private
64
30
 
65
31
  def attach_http_event_listeners(context)
66
-
67
32
  context.http_response.on_headers(200) do
68
- cipher, envelope = decryption_cipher(context)
69
- decrypter = body_contains_auth_tag?(envelope) ?
70
- authenticated_decrypter(context, cipher, envelope) :
71
- IODecrypter.new(cipher, context.http_response.body)
33
+ decrypter = if Aws::S3::EncryptionV3::Decryption.v3?(context)
34
+ cipher, envelope = Aws::S3::EncryptionV3::Decryption.decryption_cipher(context)
35
+ Aws::S3::EncryptionV3::Decryption.get_decrypter(context, cipher, envelope)
36
+ else
37
+ cipher, envelope = Aws::S3::EncryptionV2::Decryption.decryption_cipher(context)
38
+ Aws::S3::EncryptionV2::Decryption.get_decrypter(context, cipher, envelope)
39
+ end
72
40
  context.http_response.body = decrypter
73
41
  end
74
42
 
@@ -86,129 +54,6 @@ module Aws
86
54
  end
87
55
  end
88
56
 
89
- def decryption_cipher(context)
90
- if (envelope = get_encryption_envelope(context))
91
- cipher = context[:encryption][:cipher_provider]
92
- .decryption_cipher(
93
- envelope,
94
- context[:encryption]
95
- )
96
- [cipher, envelope]
97
- else
98
- raise Errors::DecryptionError, "unable to locate encryption envelope"
99
- end
100
- end
101
-
102
- def get_encryption_envelope(context)
103
- if context[:encryption][:envelope_location] == :metadata
104
- envelope_from_metadata(context) || envelope_from_instr_file(context)
105
- else
106
- envelope_from_instr_file(context) || envelope_from_metadata(context)
107
- end
108
- end
109
-
110
- def envelope_from_metadata(context)
111
- possible_envelope = {}
112
- POSSIBLE_ENVELOPE_KEYS.each do |suffix|
113
- if value = context.http_response.headers["x-amz-meta-#{suffix}"]
114
- possible_envelope[suffix] = value
115
- end
116
- end
117
- extract_envelope(possible_envelope)
118
- end
119
-
120
- def envelope_from_instr_file(context)
121
- suffix = context[:encryption][:instruction_file_suffix]
122
- possible_envelope = Json.load(context.client.get_object(
123
- bucket: context.params[:bucket],
124
- key: context.params[:key] + suffix
125
- ).body.read)
126
- extract_envelope(possible_envelope)
127
- rescue S3::Errors::ServiceError, Json::ParseError
128
- nil
129
- end
130
-
131
- def extract_envelope(hash)
132
- return nil unless hash
133
- return v1_envelope(hash) if hash.key?('x-amz-key')
134
- return v2_envelope(hash) if hash.key?('x-amz-key-v2')
135
- if hash.keys.any? { |key| key.match(/^x-amz-key-(.+)$/) }
136
- msg = "unsupported envelope encryption version #{$1}"
137
- raise Errors::DecryptionError, msg
138
- end
139
- end
140
-
141
- def v1_envelope(envelope)
142
- envelope
143
- end
144
-
145
- def v2_envelope(envelope)
146
- unless POSSIBLE_ENCRYPTION_FORMATS.include? envelope['x-amz-cek-alg']
147
- alg = envelope['x-amz-cek-alg'].inspect
148
- msg = "unsupported content encrypting key (cek) format: #{alg}"
149
- raise Errors::DecryptionError, msg
150
- end
151
- unless POSSIBLE_WRAPPING_FORMATS.include? envelope['x-amz-wrap-alg']
152
- alg = envelope['x-amz-wrap-alg'].inspect
153
- msg = "unsupported key wrapping algorithm: #{alg}"
154
- raise Errors::DecryptionError, msg
155
- end
156
- unless (missing_keys = V2_ENVELOPE_KEYS - envelope.keys).empty?
157
- msg = "incomplete v2 encryption envelope:\n"
158
- msg += " missing: #{missing_keys.join(',')}\n"
159
- raise Errors::DecryptionError, msg
160
- end
161
- envelope
162
- end
163
-
164
- # This method fetches the tag from the end of the object by
165
- # making a GET Object w/range request. This auth tag is used
166
- # to initialize the cipher, and the decrypter truncates the
167
- # auth tag from the body when writing the final bytes.
168
- def authenticated_decrypter(context, cipher, envelope)
169
- http_resp = context.http_response
170
- content_length = http_resp.headers['content-length'].to_i
171
- auth_tag_length = auth_tag_length(envelope)
172
-
173
- auth_tag = context.client.get_object(
174
- bucket: context.params[:bucket],
175
- key: context.params[:key],
176
- version_id: context.params[:version_id],
177
- range: "bytes=-#{auth_tag_length}"
178
- ).body.read
179
-
180
- cipher.auth_tag = auth_tag
181
- cipher.auth_data = ''
182
-
183
- # The encrypted object contains both the cipher text
184
- # plus a trailing auth tag.
185
- IOAuthDecrypter.new(
186
- io: http_resp.body,
187
- encrypted_content_length: content_length - auth_tag_length,
188
- cipher: cipher)
189
- end
190
-
191
- def body_contains_auth_tag?(envelope)
192
- AUTH_REQUIRED_CEK_ALGS.include?(envelope['x-amz-cek-alg'])
193
- end
194
-
195
- # Determine the auth tag length from the algorithm
196
- # Validate it against the value provided in the x-amz-tag-len
197
- # Return the tag length in bytes
198
- def auth_tag_length(envelope)
199
- tag_length =
200
- case envelope['x-amz-cek-alg']
201
- when 'AES/GCM/NoPadding' then AES_GCM_TAG_LEN_BYTES
202
- else
203
- raise ArgumentError, 'Unsupported cek-alg: ' \
204
- "#{envelope['x-amz-cek-alg']}"
205
- end
206
- if (tag_length * 8) != envelope['x-amz-tag-len'].to_i
207
- raise Errors::DecryptionError, 'x-amz-tag-len does not match expected'
208
- end
209
- tag_length
210
- end
211
-
212
57
  def apply_cse_user_agent(context)
213
58
  if context.config.user_agent_suffix.nil?
214
59
  context.config.user_agent_suffix = EC_USER_AGENT