aws-sdk-s3 1.199.1 → 1.208.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +54 -0
  3. data/VERSION +1 -1
  4. data/lib/aws-sdk-s3/bucket.rb +1 -1
  5. data/lib/aws-sdk-s3/bucket_acl.rb +1 -1
  6. data/lib/aws-sdk-s3/bucket_versioning.rb +33 -0
  7. data/lib/aws-sdk-s3/client.rb +893 -345
  8. data/lib/aws-sdk-s3/client_api.rb +61 -0
  9. data/lib/aws-sdk-s3/customizations/object.rb +26 -23
  10. data/lib/aws-sdk-s3/customizations.rb +2 -0
  11. data/lib/aws-sdk-s3/default_executor.rb +103 -0
  12. data/lib/aws-sdk-s3/encryption/client.rb +2 -2
  13. data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +2 -0
  14. data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +2 -0
  15. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +2 -0
  16. data/lib/aws-sdk-s3/encryptionV2/client.rb +98 -23
  17. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +7 -162
  18. data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
  19. data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +17 -0
  20. data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +2 -0
  21. data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +2 -0
  22. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +8 -0
  23. data/lib/aws-sdk-s3/encryptionV2/utils.rb +5 -0
  24. data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
  25. data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
  26. data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
  27. data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
  28. data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
  29. data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
  30. data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
  31. data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
  32. data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
  33. data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
  34. data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
  35. data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
  36. data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
  37. data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
  38. data/lib/aws-sdk-s3/encryption_v2.rb +1 -0
  39. data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
  40. data/lib/aws-sdk-s3/endpoint_parameters.rb +17 -17
  41. data/lib/aws-sdk-s3/endpoint_provider.rb +220 -50
  42. data/lib/aws-sdk-s3/endpoints.rb +26 -0
  43. data/lib/aws-sdk-s3/file_downloader.rb +192 -112
  44. data/lib/aws-sdk-s3/file_uploader.rb +6 -8
  45. data/lib/aws-sdk-s3/multipart_file_uploader.rb +79 -63
  46. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +41 -44
  47. data/lib/aws-sdk-s3/object.rb +56 -27
  48. data/lib/aws-sdk-s3/object_acl.rb +1 -1
  49. data/lib/aws-sdk-s3/object_summary.rb +42 -13
  50. data/lib/aws-sdk-s3/object_version.rb +7 -9
  51. data/lib/aws-sdk-s3/plugins/endpoints.rb +1 -1
  52. data/lib/aws-sdk-s3/transfer_manager.rb +80 -29
  53. data/lib/aws-sdk-s3/types.rb +379 -215
  54. data/lib/aws-sdk-s3.rb +1 -1
  55. data/sig/bucket.rbs +1 -1
  56. data/sig/client.rbs +40 -12
  57. data/sig/multipart_upload.rbs +1 -1
  58. data/sig/object.rbs +7 -5
  59. data/sig/object_summary.rbs +7 -5
  60. data/sig/types.rbs +47 -14
  61. metadata +20 -3
@@ -14,6 +14,7 @@ module Aws::S3
14
14
 
15
15
  include Seahorse::Model
16
16
 
17
+ AbacStatus = Shapes::StructureShape.new(name: 'AbacStatus')
17
18
  AbortDate = Shapes::TimestampShape.new(name: 'AbortDate')
18
19
  AbortIncompleteMultipartUpload = Shapes::StructureShape.new(name: 'AbortIncompleteMultipartUpload')
19
20
  AbortMultipartUploadOutput = Shapes::StructureShape.new(name: 'AbortMultipartUploadOutput')
@@ -43,8 +44,10 @@ module Aws::S3
43
44
  AnalyticsS3BucketDestination = Shapes::StructureShape.new(name: 'AnalyticsS3BucketDestination')
44
45
  AnalyticsS3ExportFileFormat = Shapes::StringShape.new(name: 'AnalyticsS3ExportFileFormat')
45
46
  ArchiveStatus = Shapes::StringShape.new(name: 'ArchiveStatus')
47
+ BlockedEncryptionTypes = Shapes::StructureShape.new(name: 'BlockedEncryptionTypes')
46
48
  Body = Shapes::BlobShape.new(name: 'Body')
47
49
  Bucket = Shapes::StructureShape.new(name: 'Bucket')
50
+ BucketAbacStatus = Shapes::StringShape.new(name: 'BucketAbacStatus')
48
51
  BucketAccelerateStatus = Shapes::StringShape.new(name: 'BucketAccelerateStatus')
49
52
  BucketAlreadyExists = Shapes::StructureShape.new(name: 'BucketAlreadyExists')
50
53
  BucketAlreadyOwnedByYou = Shapes::StructureShape.new(name: 'BucketAlreadyOwnedByYou')
@@ -177,6 +180,8 @@ module Aws::S3
177
180
  EncodingType = Shapes::StringShape.new(name: 'EncodingType')
178
181
  Encryption = Shapes::StructureShape.new(name: 'Encryption')
179
182
  EncryptionConfiguration = Shapes::StructureShape.new(name: 'EncryptionConfiguration')
183
+ EncryptionType = Shapes::StringShape.new(name: 'EncryptionType')
184
+ EncryptionTypeList = Shapes::ListShape.new(name: 'EncryptionTypeList', flattened: true)
180
185
  EncryptionTypeMismatch = Shapes::StructureShape.new(name: 'EncryptionTypeMismatch')
181
186
  End = Shapes::IntegerShape.new(name: 'End')
182
187
  EndEvent = Shapes::StructureShape.new(name: 'EndEvent')
@@ -208,6 +213,8 @@ module Aws::S3
208
213
  FilterRuleList = Shapes::ListShape.new(name: 'FilterRuleList', flattened: true)
209
214
  FilterRuleName = Shapes::StringShape.new(name: 'FilterRuleName')
210
215
  FilterRuleValue = Shapes::StringShape.new(name: 'FilterRuleValue')
216
+ GetBucketAbacOutput = Shapes::StructureShape.new(name: 'GetBucketAbacOutput')
217
+ GetBucketAbacRequest = Shapes::StructureShape.new(name: 'GetBucketAbacRequest')
211
218
  GetBucketAccelerateConfigurationOutput = Shapes::StructureShape.new(name: 'GetBucketAccelerateConfigurationOutput')
212
219
  GetBucketAccelerateConfigurationRequest = Shapes::StructureShape.new(name: 'GetBucketAccelerateConfigurationRequest')
213
220
  GetBucketAclOutput = Shapes::StructureShape.new(name: 'GetBucketAclOutput')
@@ -497,6 +504,7 @@ module Aws::S3
497
504
  ProgressEvent = Shapes::StructureShape.new(name: 'ProgressEvent')
498
505
  Protocol = Shapes::StringShape.new(name: 'Protocol')
499
506
  PublicAccessBlockConfiguration = Shapes::StructureShape.new(name: 'PublicAccessBlockConfiguration')
507
+ PutBucketAbacRequest = Shapes::StructureShape.new(name: 'PutBucketAbacRequest')
500
508
  PutBucketAccelerateConfigurationRequest = Shapes::StructureShape.new(name: 'PutBucketAccelerateConfigurationRequest')
501
509
  PutBucketAclRequest = Shapes::StructureShape.new(name: 'PutBucketAclRequest')
502
510
  PutBucketAnalyticsConfigurationRequest = Shapes::StructureShape.new(name: 'PutBucketAnalyticsConfigurationRequest')
@@ -686,6 +694,9 @@ module Aws::S3
686
694
  WriteOffsetBytes = Shapes::IntegerShape.new(name: 'WriteOffsetBytes')
687
695
  Years = Shapes::IntegerShape.new(name: 'Years')
688
696
 
697
+ AbacStatus.add_member(:status, Shapes::ShapeRef.new(shape: BucketAbacStatus, location_name: "Status"))
698
+ AbacStatus.struct_class = Types::AbacStatus
699
+
689
700
  AbortIncompleteMultipartUpload.add_member(:days_after_initiation, Shapes::ShapeRef.new(shape: DaysAfterInitiation, location_name: "DaysAfterInitiation"))
690
701
  AbortIncompleteMultipartUpload.struct_class = Types::AbortIncompleteMultipartUpload
691
702
 
@@ -741,6 +752,9 @@ module Aws::S3
741
752
  AnalyticsS3BucketDestination.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, location_name: "Prefix"))
742
753
  AnalyticsS3BucketDestination.struct_class = Types::AnalyticsS3BucketDestination
743
754
 
755
+ BlockedEncryptionTypes.add_member(:encryption_type, Shapes::ShapeRef.new(shape: EncryptionTypeList, location_name: "EncryptionType"))
756
+ BlockedEncryptionTypes.struct_class = Types::BlockedEncryptionTypes
757
+
744
758
  Bucket.add_member(:name, Shapes::ShapeRef.new(shape: BucketName, location_name: "Name"))
745
759
  Bucket.add_member(:creation_date, Shapes::ShapeRef.new(shape: CreationDate, location_name: "CreationDate"))
746
760
  Bucket.add_member(:bucket_region, Shapes::ShapeRef.new(shape: BucketRegion, location_name: "BucketRegion"))
@@ -907,6 +921,8 @@ module Aws::S3
907
921
  CopyObjectRequest.add_member(:grant_read, Shapes::ShapeRef.new(shape: GrantRead, location: "header", location_name: "x-amz-grant-read"))
908
922
  CopyObjectRequest.add_member(:grant_read_acp, Shapes::ShapeRef.new(shape: GrantReadACP, location: "header", location_name: "x-amz-grant-read-acp"))
909
923
  CopyObjectRequest.add_member(:grant_write_acp, Shapes::ShapeRef.new(shape: GrantWriteACP, location: "header", location_name: "x-amz-grant-write-acp"))
924
+ CopyObjectRequest.add_member(:if_match, Shapes::ShapeRef.new(shape: IfMatch, location: "header", location_name: "If-Match"))
925
+ CopyObjectRequest.add_member(:if_none_match, Shapes::ShapeRef.new(shape: IfNoneMatch, location: "header", location_name: "If-None-Match"))
910
926
  CopyObjectRequest.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, required: true, location: "uri", location_name: "Key", metadata: {"contextParam" => {"name" => "Key"}}))
911
927
  CopyObjectRequest.add_member(:metadata, Shapes::ShapeRef.new(shape: Metadata, location: "headers", location_name: "x-amz-meta-"))
912
928
  CopyObjectRequest.add_member(:metadata_directive, Shapes::ShapeRef.new(shape: MetadataDirective, location: "header", location_name: "x-amz-metadata-directive"))
@@ -1218,6 +1234,8 @@ module Aws::S3
1218
1234
  EncryptionConfiguration.add_member(:replica_kms_key_id, Shapes::ShapeRef.new(shape: ReplicaKmsKeyID, location_name: "ReplicaKmsKeyID"))
1219
1235
  EncryptionConfiguration.struct_class = Types::EncryptionConfiguration
1220
1236
 
1237
+ EncryptionTypeList.member = Shapes::ShapeRef.new(shape: EncryptionType, location_name: "EncryptionType")
1238
+
1221
1239
  EncryptionTypeMismatch.struct_class = Types::EncryptionTypeMismatch
1222
1240
 
1223
1241
  EndEvent.struct_class = Types::EndEvent
@@ -1252,6 +1270,15 @@ module Aws::S3
1252
1270
 
1253
1271
  FilterRuleList.member = Shapes::ShapeRef.new(shape: FilterRule)
1254
1272
 
1273
+ GetBucketAbacOutput.add_member(:abac_status, Shapes::ShapeRef.new(shape: AbacStatus, location_name: "AbacStatus"))
1274
+ GetBucketAbacOutput.struct_class = Types::GetBucketAbacOutput
1275
+ GetBucketAbacOutput[:payload] = :abac_status
1276
+ GetBucketAbacOutput[:payload_member] = GetBucketAbacOutput.member(:abac_status)
1277
+
1278
+ GetBucketAbacRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
1279
+ GetBucketAbacRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
1280
+ GetBucketAbacRequest.struct_class = Types::GetBucketAbacRequest
1281
+
1255
1282
  GetBucketAccelerateConfigurationOutput.add_member(:status, Shapes::ShapeRef.new(shape: BucketAccelerateStatus, location_name: "Status"))
1256
1283
  GetBucketAccelerateConfigurationOutput.add_member(:request_charged, Shapes::ShapeRef.new(shape: RequestCharged, location: "header", location_name: "x-amz-request-charged"))
1257
1284
  GetBucketAccelerateConfigurationOutput.struct_class = Types::GetBucketAccelerateConfigurationOutput
@@ -2292,6 +2319,15 @@ module Aws::S3
2292
2319
  PublicAccessBlockConfiguration.add_member(:restrict_public_buckets, Shapes::ShapeRef.new(shape: Setting, location_name: "RestrictPublicBuckets"))
2293
2320
  PublicAccessBlockConfiguration.struct_class = Types::PublicAccessBlockConfiguration
2294
2321
 
2322
+ PutBucketAbacRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
2323
+ PutBucketAbacRequest.add_member(:content_md5, Shapes::ShapeRef.new(shape: ContentMD5, location: "header", location_name: "Content-MD5"))
2324
+ PutBucketAbacRequest.add_member(:checksum_algorithm, Shapes::ShapeRef.new(shape: ChecksumAlgorithm, location: "header", location_name: "x-amz-sdk-checksum-algorithm"))
2325
+ PutBucketAbacRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
2326
+ PutBucketAbacRequest.add_member(:abac_status, Shapes::ShapeRef.new(shape: AbacStatus, required: true, location_name: "AbacStatus", metadata: {"xmlNamespace" => {"uri" => "http://s3.amazonaws.com/doc/2006-03-01/"}}))
2327
+ PutBucketAbacRequest.struct_class = Types::PutBucketAbacRequest
2328
+ PutBucketAbacRequest[:payload] = :abac_status
2329
+ PutBucketAbacRequest[:payload_member] = PutBucketAbacRequest.member(:abac_status)
2330
+
2295
2331
  PutBucketAccelerateConfigurationRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
2296
2332
  PutBucketAccelerateConfigurationRequest.add_member(:accelerate_configuration, Shapes::ShapeRef.new(shape: AccelerateConfiguration, required: true, location_name: "AccelerateConfiguration", metadata: {"xmlNamespace" => {"uri" => "http://s3.amazonaws.com/doc/2006-03-01/"}}))
2297
2333
  PutBucketAccelerateConfigurationRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
@@ -2841,6 +2877,7 @@ module Aws::S3
2841
2877
 
2842
2878
  ServerSideEncryptionRule.add_member(:apply_server_side_encryption_by_default, Shapes::ShapeRef.new(shape: ServerSideEncryptionByDefault, location_name: "ApplyServerSideEncryptionByDefault"))
2843
2879
  ServerSideEncryptionRule.add_member(:bucket_key_enabled, Shapes::ShapeRef.new(shape: BucketKeyEnabled, location_name: "BucketKeyEnabled"))
2880
+ ServerSideEncryptionRule.add_member(:blocked_encryption_types, Shapes::ShapeRef.new(shape: BlockedEncryptionTypes, location_name: "BlockedEncryptionTypes"))
2844
2881
  ServerSideEncryptionRule.struct_class = Types::ServerSideEncryptionRule
2845
2882
 
2846
2883
  ServerSideEncryptionRules.member = Shapes::ShapeRef.new(shape: ServerSideEncryptionRule)
@@ -3332,6 +3369,14 @@ module Aws::S3
3332
3369
  o.output = Shapes::ShapeRef.new(shape: Shapes::StructureShape.new(struct_class: Aws::EmptyStructure))
3333
3370
  end)
3334
3371
 
3372
+ api.add_operation(:get_bucket_abac, Seahorse::Model::Operation.new.tap do |o|
3373
+ o.name = "GetBucketAbac"
3374
+ o.http_method = "GET"
3375
+ o.http_request_uri = "/?abac"
3376
+ o.input = Shapes::ShapeRef.new(shape: GetBucketAbacRequest)
3377
+ o.output = Shapes::ShapeRef.new(shape: GetBucketAbacOutput)
3378
+ end)
3379
+
3335
3380
  api.add_operation(:get_bucket_accelerate_configuration, Seahorse::Model::Operation.new.tap do |o|
3336
3381
  o.name = "GetBucketAccelerateConfiguration"
3337
3382
  o.http_method = "GET"
@@ -3766,6 +3811,22 @@ module Aws::S3
3766
3811
  )
3767
3812
  end)
3768
3813
 
3814
+ api.add_operation(:put_bucket_abac, Seahorse::Model::Operation.new.tap do |o|
3815
+ o.name = "PutBucketAbac"
3816
+ o.http_method = "PUT"
3817
+ o.http_request_uri = "/?abac"
3818
+ o.http_checksum = {
3819
+ "requestAlgorithmMember" => "checksum_algorithm",
3820
+ "requestChecksumRequired" => false,
3821
+ }
3822
+ o.http_checksum = {
3823
+ "requestAlgorithmMember" => "checksum_algorithm",
3824
+ "requestChecksumRequired" => false,
3825
+ }
3826
+ o.input = Shapes::ShapeRef.new(shape: PutBucketAbacRequest)
3827
+ o.output = Shapes::ShapeRef.new(shape: Shapes::StructureShape.new(struct_class: Aws::EmptyStructure))
3828
+ end)
3829
+
3769
3830
  api.add_operation(:put_bucket_accelerate_configuration, Seahorse::Model::Operation.new.tap do |o|
3770
3831
  o.name = "PutBucketAccelerateConfiguration"
3771
3832
  o.http_method = "PUT"
@@ -358,8 +358,8 @@ module Aws
358
358
  # {Client#complete_multipart_upload},
359
359
  # and {Client#upload_part} can be provided.
360
360
  #
361
- # @option options [Integer] :thread_count (10) The number of parallel
362
- # multipart uploads
361
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
362
+ # An additional thread is used internally for task coordination.
363
363
  #
364
364
  # @option options [Boolean] :tempfile (false) Normally read data is stored
365
365
  # in memory when building the parts in order to complete the underlying
@@ -383,19 +383,18 @@ module Aws
383
383
  # @see Client#complete_multipart_upload
384
384
  # @see Client#upload_part
385
385
  def upload_stream(options = {}, &block)
386
- uploading_options = options.dup
386
+ upload_opts = options.merge(bucket: bucket_name, key: key)
387
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
387
388
  uploader = MultipartStreamUploader.new(
388
389
  client: client,
389
- thread_count: uploading_options.delete(:thread_count),
390
- tempfile: uploading_options.delete(:tempfile),
391
- part_size: uploading_options.delete(:part_size)
390
+ executor: executor,
391
+ tempfile: upload_opts.delete(:tempfile),
392
+ part_size: upload_opts.delete(:part_size)
392
393
  )
393
394
  Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
394
- uploader.upload(
395
- uploading_options.merge(bucket: bucket_name, key: key),
396
- &block
397
- )
395
+ uploader.upload(upload_opts, &block)
398
396
  end
397
+ executor.shutdown
399
398
  true
400
399
  end
401
400
  deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
@@ -458,12 +457,18 @@ module Aws
458
457
  # @see Client#complete_multipart_upload
459
458
  # @see Client#upload_part
460
459
  def upload_file(source, options = {})
461
- uploading_options = options.dup
462
- uploader = FileUploader.new(multipart_threshold: uploading_options.delete(:multipart_threshold), client: client)
460
+ upload_opts = options.merge(bucket: bucket_name, key: key)
461
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
462
+ uploader = FileUploader.new(
463
+ client: client,
464
+ executor: executor,
465
+ multipart_threshold: upload_opts.delete(:multipart_threshold)
466
+ )
463
467
  response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
464
- uploader.upload(source, uploading_options.merge(bucket: bucket_name, key: key))
468
+ uploader.upload(source, upload_opts)
465
469
  end
466
470
  yield response if block_given?
471
+ executor.shutdown
467
472
  true
468
473
  end
469
474
  deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
@@ -512,15 +517,10 @@ module Aws
512
517
  #
513
518
  # @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
514
519
  #
515
- # @option options [String] :version_id The object version id used to retrieve the object.
516
- #
517
- # @see https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html ObjectVersioning
518
- #
519
520
  # @option options [String] :checksum_mode ("ENABLED")
520
- # When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will
521
- # raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated`
522
- # callback if you need to verify that validation occurred and which algorithm was used.
523
- # To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
521
+ # This option is deprecated. Use `:response_checksum_validation` on your S3 client instead.
522
+ # To disable checksum validation, set `response_checksum_validation: 'when_required'`
523
+ # when creating your S3 client.
524
524
  #
525
525
  # @option options [Callable] :on_checksum_validated
526
526
  # Called each time a request's checksum is validated with the checksum algorithm and the
@@ -539,10 +539,13 @@ module Aws
539
539
  # @see Client#get_object
540
540
  # @see Client#head_object
541
541
  def download_file(destination, options = {})
542
- downloader = FileDownloader.new(client: client)
542
+ download_opts = options.merge(bucket: bucket_name, key: key)
543
+ executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
544
+ downloader = FileDownloader.new(client: client, executor: executor)
543
545
  Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
544
- downloader.download(destination, options.merge(bucket: bucket_name, key: key))
546
+ downloader.download(destination, download_opts)
545
547
  end
548
+ executor.shutdown
546
549
  true
547
550
  end
548
551
  deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
@@ -6,7 +6,9 @@ module Aws
6
6
  autoload :BucketRegionCache, 'aws-sdk-s3/bucket_region_cache'
7
7
  autoload :Encryption, 'aws-sdk-s3/encryption'
8
8
  autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
9
+ autoload :EncryptionV3, 'aws-sdk-s3/encryption_v3'
9
10
  autoload :FilePart, 'aws-sdk-s3/file_part'
11
+ autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
10
12
  autoload :FileUploader, 'aws-sdk-s3/file_uploader'
11
13
  autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
12
14
  autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
@@ -0,0 +1,103 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ # @api private
6
+ class DefaultExecutor
7
+ DEFAULT_MAX_THREADS = 10
8
+ RUNNING = :running
9
+ SHUTTING_DOWN = :shutting_down
10
+ SHUTDOWN = :shutdown
11
+
12
+ def initialize(options = {})
13
+ @max_threads = options[:max_threads] || DEFAULT_MAX_THREADS
14
+ @state = RUNNING
15
+ @queue = Queue.new
16
+ @pool = []
17
+ @mutex = Mutex.new
18
+ end
19
+
20
+ # Submits a task for execution.
21
+ # @param [Object] args Variable number of arguments to pass to the block
22
+ # @param [Proc] block The block to be executed
23
+ # @return [Boolean] Returns true if the task was submitted successfully
24
+ def post(*args, &block)
25
+ @mutex.synchronize do
26
+ raise 'Executor has been shutdown and is no longer accepting tasks' unless @state == RUNNING
27
+
28
+ @queue << [args, block]
29
+ ensure_worker_available
30
+ end
31
+ true
32
+ end
33
+
34
+ # Immediately terminates all worker threads and clears pending tasks.
35
+ # This is a forceful shutdown that doesn't wait for running tasks to complete.
36
+ #
37
+ # @return [Boolean] true when termination is complete
38
+ def kill
39
+ @mutex.synchronize do
40
+ @state = SHUTDOWN
41
+ @pool.each(&:kill)
42
+ @pool.clear
43
+ @queue.clear
44
+ end
45
+ true
46
+ end
47
+
48
+ # Gracefully shuts down the executor, optionally with a timeout.
49
+ # Stops accepting new tasks and waits for running tasks to complete.
50
+ #
51
+ # @param timeout [Numeric, nil] Maximum time in seconds to wait for shutdown.
52
+ # If nil, waits indefinitely. If timeout expires, remaining threads are killed.
53
+ # @return [Boolean] true when shutdown is complete
54
+ def shutdown(timeout = nil)
55
+ @mutex.synchronize do
56
+ return true if @state == SHUTDOWN
57
+
58
+ @state = SHUTTING_DOWN
59
+ @pool.size.times { @queue << :shutdown }
60
+ end
61
+
62
+ if timeout
63
+ deadline = Time.now + timeout
64
+ @pool.each do |thread|
65
+ remaining = deadline - Time.now
66
+ break if remaining <= 0
67
+
68
+ thread.join([remaining, 0].max)
69
+ end
70
+ @pool.select(&:alive?).each(&:kill)
71
+ else
72
+ @pool.each(&:join)
73
+ end
74
+
75
+ @mutex.synchronize do
76
+ @pool.clear
77
+ @state = SHUTDOWN
78
+ end
79
+ true
80
+ end
81
+
82
+ private
83
+
84
+ def ensure_worker_available
85
+ return unless @state == RUNNING
86
+
87
+ @pool.select!(&:alive?)
88
+ @pool << spawn_worker if @pool.size < @max_threads
89
+ end
90
+
91
+ def spawn_worker
92
+ Thread.new do
93
+ while (job = @queue.shift)
94
+ break if job == :shutdown
95
+
96
+ args, block = job
97
+ block.call(*args)
98
+ end
99
+ end
100
+ end
101
+ end
102
+ end
103
+ end
@@ -6,9 +6,9 @@ module Aws
6
6
  module S3
7
7
 
8
8
  # [MAINTENANCE MODE] There is a new version of the Encryption Client.
9
- # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV2::Client},
9
+ # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
10
10
  # which provides updated data security best practices.
11
- # See documentation for {Aws::S3::EncryptionV2::Client}.
11
+ # See documentation for {Aws::S3::EncryptionV3::Client}.
12
12
  # Provides an encryption client that encrypts and decrypts data client-side,
13
13
  # storing the encrypted data in Amazon S3.
14
14
  #
@@ -16,6 +16,8 @@ module Aws
16
16
  # envelope and encryption cipher.
17
17
  def encryption_cipher
18
18
  cipher = Utils.aes_encryption_cipher(:CBC)
19
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
20
+ ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
19
21
  envelope = {
20
22
  'x-amz-key' => encode64(encrypt(envelope_key(cipher))),
21
23
  'x-amz-iv' => encode64(envelope_iv(cipher)),
@@ -38,6 +38,8 @@ module Aws
38
38
  io = StringIO.new(io) if String === io
39
39
  context.params[:body] = IOEncrypter.new(cipher, io)
40
40
  context.params[:metadata] ||= {}
41
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys
42
+ ##% - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects.
41
43
  context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
42
44
  if context.params.delete(:content_md5)
43
45
  warn('Setting content_md5 on client side encrypted objects is deprecated')
@@ -26,6 +26,8 @@ module Aws
26
26
  end
27
27
  cipher = Utils.aes_encryption_cipher(:CBC)
28
28
  cipher.key = key_data.plaintext
29
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
30
+ ##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
29
31
  envelope = {
30
32
  'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
31
33
  'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
@@ -5,9 +5,17 @@ require 'forwardable'
5
5
  module Aws
6
6
  module S3
7
7
 
8
- REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile]
9
- SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy]
8
+ REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile].freeze
9
+ SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy].freeze
10
+ SUPPORTED_COMMITMENT_POLICIES = [:forbid_encrypt_allow_decrypt].freeze
10
11
 
12
+ # [MAINTENANCE MODE] There is a new version of the Encryption Client.
13
+ # AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
14
+ # which provides updated data security best practices.
15
+ # For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
16
+ # Provides an encryption client that encrypts and decrypts data client-side,
17
+ # storing the encrypted data in Amazon S3.
18
+ #
11
19
  # Provides an encryption client that encrypts and decrypts data client-side,
12
20
  # storing the encrypted data in Amazon S3. The `EncryptionV2::Client` (V2 Client)
13
21
  # provides improved security over the `Encryption::Client` (V1 Client)
@@ -307,15 +315,29 @@ module Aws
307
315
  # @option options [KMS::Client] :kms_client A default {KMS::Client}
308
316
  # is constructed when using KMS to manage encryption keys.
309
317
  #
318
+ # @option options [Symbol] :commitment_policy (nil)
319
+ # Optional parameter for migration from V2 to V3. When set to
320
+ # :forbid_encrypt_allow_decrypt, this explicitly indicates you are
321
+ # maintaining V2 encryption behavior while preparing for migration.
322
+ # This allows the V2 client to decrypt V3-encrypted objects while
323
+ # continuing to encrypt new objects using V2 algorithms.
324
+ # Only :forbid_encrypt_allow_decrypt is supported.
325
+ # For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
326
+ #
310
327
  def initialize(options = {})
311
328
  validate_params(options)
312
329
  @client = extract_client(options)
313
- @cipher_provider = cipher_provider(options)
330
+ @cipher_provider = build_cipher_provider(options)
331
+ @key_provider = @cipher_provider.key_provider if @cipher_provider.is_a?(DefaultCipherProvider)
314
332
  @envelope_location = extract_location(options)
315
333
  @instruction_file_suffix = extract_suffix(options)
316
334
  @kms_allow_decrypt_with_any_cmk =
317
335
  options[:kms_key_id] == :kms_allow_decrypt_with_any_cmk
318
336
  @security_profile = extract_security_profile(options)
337
+ @commitment_policy = extract_commitment_policy(options)
338
+ # The v3 cipher is only used for decrypt.
339
+ # Therefore any configured v2 `content_encryption_schema` is going to be incorrect.
340
+ @v3_cipher_provider = build_v3_cipher_provider_for_decrypt(options.reject { |k, _| k == :content_encryption_schema })
319
341
  end
320
342
 
321
343
  # @return [S3::Client]
@@ -341,6 +363,11 @@ module Aws
341
363
  # by this string.
342
364
  attr_reader :instruction_file_suffix
343
365
 
366
+ # @return [Symbol, nil] Optional commitment policy for V2 to V3 migration.
367
+ # When set to :forbid_encrypt_allow_decrypt, explicitly indicates
368
+ # maintaining V2 encryption behavior while preparing for migration.
369
+ attr_reader :commitment_policy
370
+
344
371
  # Uploads an object to Amazon S3, encrypting data client-side.
345
372
  # See {S3::Client#put_object} for documentation on accepted
346
373
  # request parameters.
@@ -410,6 +437,7 @@ module Aws
410
437
  req.handlers.add(DecryptHandler)
411
438
  req.context[:encryption] = {
412
439
  cipher_provider: @cipher_provider,
440
+ v3_cipher_provider: @v3_cipher_provider,
413
441
  envelope_location: envelope_location,
414
442
  instruction_file_suffix: instruction_file_suffix,
415
443
  kms_encryption_context: kms_encryption_context,
@@ -423,6 +451,50 @@ module Aws
423
451
 
424
452
  private
425
453
 
454
+ def build_cipher_provider(options)
455
+ if options[:kms_key_id]
456
+ KmsCipherProvider.new(
457
+ kms_key_id: options[:kms_key_id],
458
+ kms_client: kms_client(options),
459
+ key_wrap_schema: options[:key_wrap_schema],
460
+ content_encryption_schema: options[:content_encryption_schema]
461
+ )
462
+ else
463
+ key_provider = extract_key_provider(options)
464
+ DefaultCipherProvider.new(
465
+ key_provider: key_provider,
466
+ key_wrap_schema: options[:key_wrap_schema],
467
+ content_encryption_schema: options[:content_encryption_schema]
468
+ )
469
+ end
470
+ end
471
+
472
+ def build_v3_cipher_provider_for_decrypt(options)
473
+ if options[:kms_key_id]
474
+ Aws::S3::EncryptionV3::KmsCipherProvider.new(
475
+ kms_key_id: options[:kms_key_id],
476
+ kms_client: kms_client(options),
477
+ key_wrap_schema: options[:key_wrap_schema],
478
+ content_encryption_schema: options[:content_encryption_schema]
479
+ )
480
+ else
481
+ # Create V3 key provider explicitly for proper namespace consistency
482
+ key_provider = if options[:key_provider]
483
+ options[:key_provider]
484
+ elsif options[:encryption_key]
485
+ Aws::S3::EncryptionV3::DefaultKeyProvider.new(options)
486
+ else
487
+ msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
488
+ raise ArgumentError, msg
489
+ end
490
+ Aws::S3::EncryptionV3::DefaultCipherProvider.new(
491
+ key_provider: key_provider,
492
+ key_wrap_schema: options[:key_wrap_schema],
493
+ content_encryption_schema: options[:content_encryption_schema]
494
+ )
495
+ end
496
+ end
497
+
426
498
  # Validate required parameters exist and don't conflict.
427
499
  # The cek_alg and wrap_alg are passed on to the CipherProviders
428
500
  # and further validated there
@@ -452,36 +524,19 @@ module Aws
452
524
  options.delete(:encryption_key)
453
525
  options.delete(:envelope_location)
454
526
  options.delete(:instruction_file_suffix)
527
+ options.delete(:commitment_policy)
455
528
  REQUIRED_PARAMS.each { |p| options.delete(p) }
456
529
  S3::Client.new(options)
457
530
  end
458
531
  end
459
532
 
460
533
  def kms_client(options)
461
- options[:kms_client] || begin
534
+ options[:kms_client] || (@kms_client ||=
462
535
  KMS::Client.new(
463
536
  region: @client.config.region,
464
537
  credentials: @client.config.credentials,
465
538
  )
466
- end
467
- end
468
-
469
- def cipher_provider(options)
470
- if options[:kms_key_id]
471
- KmsCipherProvider.new(
472
- kms_key_id: options[:kms_key_id],
473
- kms_client: kms_client(options),
474
- key_wrap_schema: options[:key_wrap_schema],
475
- content_encryption_schema: options[:content_encryption_schema]
476
- )
477
- else
478
- @key_provider = extract_key_provider(options)
479
- DefaultCipherProvider.new(
480
- key_provider: @key_provider,
481
- key_wrap_schema: options[:key_wrap_schema],
482
- content_encryption_schema: options[:content_encryption_schema]
483
- )
484
- end
539
+ )
485
540
  end
486
541
 
487
542
  def extract_key_provider(options)
@@ -564,7 +619,27 @@ module Aws
564
619
  end
565
620
  security_profile
566
621
  end
622
+
623
+ def extract_commitment_policy(options)
624
+ validate_commitment_policy(options[:commitment_policy])
625
+ end
626
+
627
+ def validate_commitment_policy(commitment_policy)
628
+ return nil if commitment_policy.nil?
629
+
630
+ unless SUPPORTED_COMMITMENT_POLICIES.include? commitment_policy
631
+ raise ArgumentError, "Unsupported commitment policy: :#{commitment_policy}. " \
632
+ "The V2 client only supports :forbid_encrypt_allow_decrypt for migration purposes. " \
633
+ "For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html"
634
+ end
635
+ commitment_policy
636
+ end
567
637
  end
568
638
  end
569
639
  end
570
640
  end
641
+
642
+ ##= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared
643
+ ##= type=exception
644
+ ##= reason=This has never been supported in Ruby
645
+ ##% This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server.