aws-sdk-s3 1.199.1 → 1.208.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +54 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-s3/bucket.rb +1 -1
- data/lib/aws-sdk-s3/bucket_acl.rb +1 -1
- data/lib/aws-sdk-s3/bucket_versioning.rb +33 -0
- data/lib/aws-sdk-s3/client.rb +893 -345
- data/lib/aws-sdk-s3/client_api.rb +61 -0
- data/lib/aws-sdk-s3/customizations/object.rb +26 -23
- data/lib/aws-sdk-s3/customizations.rb +2 -0
- data/lib/aws-sdk-s3/default_executor.rb +103 -0
- data/lib/aws-sdk-s3/encryption/client.rb +2 -2
- data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +2 -0
- data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +2 -0
- data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/client.rb +98 -23
- data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +7 -162
- data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
- data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +17 -0
- data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +2 -0
- data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +8 -0
- data/lib/aws-sdk-s3/encryptionV2/utils.rb +5 -0
- data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
- data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
- data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
- data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
- data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
- data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
- data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
- data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
- data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
- data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
- data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
- data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
- data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
- data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
- data/lib/aws-sdk-s3/encryption_v2.rb +1 -0
- data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
- data/lib/aws-sdk-s3/endpoint_parameters.rb +17 -17
- data/lib/aws-sdk-s3/endpoint_provider.rb +220 -50
- data/lib/aws-sdk-s3/endpoints.rb +26 -0
- data/lib/aws-sdk-s3/file_downloader.rb +192 -112
- data/lib/aws-sdk-s3/file_uploader.rb +6 -8
- data/lib/aws-sdk-s3/multipart_file_uploader.rb +79 -63
- data/lib/aws-sdk-s3/multipart_stream_uploader.rb +41 -44
- data/lib/aws-sdk-s3/object.rb +56 -27
- data/lib/aws-sdk-s3/object_acl.rb +1 -1
- data/lib/aws-sdk-s3/object_summary.rb +42 -13
- data/lib/aws-sdk-s3/object_version.rb +7 -9
- data/lib/aws-sdk-s3/plugins/endpoints.rb +1 -1
- data/lib/aws-sdk-s3/transfer_manager.rb +80 -29
- data/lib/aws-sdk-s3/types.rb +379 -215
- data/lib/aws-sdk-s3.rb +1 -1
- data/sig/bucket.rbs +1 -1
- data/sig/client.rbs +40 -12
- data/sig/multipart_upload.rbs +1 -1
- data/sig/object.rbs +7 -5
- data/sig/object_summary.rbs +7 -5
- data/sig/types.rbs +47 -14
- metadata +20 -3
|
@@ -14,6 +14,7 @@ module Aws::S3
|
|
|
14
14
|
|
|
15
15
|
include Seahorse::Model
|
|
16
16
|
|
|
17
|
+
AbacStatus = Shapes::StructureShape.new(name: 'AbacStatus')
|
|
17
18
|
AbortDate = Shapes::TimestampShape.new(name: 'AbortDate')
|
|
18
19
|
AbortIncompleteMultipartUpload = Shapes::StructureShape.new(name: 'AbortIncompleteMultipartUpload')
|
|
19
20
|
AbortMultipartUploadOutput = Shapes::StructureShape.new(name: 'AbortMultipartUploadOutput')
|
|
@@ -43,8 +44,10 @@ module Aws::S3
|
|
|
43
44
|
AnalyticsS3BucketDestination = Shapes::StructureShape.new(name: 'AnalyticsS3BucketDestination')
|
|
44
45
|
AnalyticsS3ExportFileFormat = Shapes::StringShape.new(name: 'AnalyticsS3ExportFileFormat')
|
|
45
46
|
ArchiveStatus = Shapes::StringShape.new(name: 'ArchiveStatus')
|
|
47
|
+
BlockedEncryptionTypes = Shapes::StructureShape.new(name: 'BlockedEncryptionTypes')
|
|
46
48
|
Body = Shapes::BlobShape.new(name: 'Body')
|
|
47
49
|
Bucket = Shapes::StructureShape.new(name: 'Bucket')
|
|
50
|
+
BucketAbacStatus = Shapes::StringShape.new(name: 'BucketAbacStatus')
|
|
48
51
|
BucketAccelerateStatus = Shapes::StringShape.new(name: 'BucketAccelerateStatus')
|
|
49
52
|
BucketAlreadyExists = Shapes::StructureShape.new(name: 'BucketAlreadyExists')
|
|
50
53
|
BucketAlreadyOwnedByYou = Shapes::StructureShape.new(name: 'BucketAlreadyOwnedByYou')
|
|
@@ -177,6 +180,8 @@ module Aws::S3
|
|
|
177
180
|
EncodingType = Shapes::StringShape.new(name: 'EncodingType')
|
|
178
181
|
Encryption = Shapes::StructureShape.new(name: 'Encryption')
|
|
179
182
|
EncryptionConfiguration = Shapes::StructureShape.new(name: 'EncryptionConfiguration')
|
|
183
|
+
EncryptionType = Shapes::StringShape.new(name: 'EncryptionType')
|
|
184
|
+
EncryptionTypeList = Shapes::ListShape.new(name: 'EncryptionTypeList', flattened: true)
|
|
180
185
|
EncryptionTypeMismatch = Shapes::StructureShape.new(name: 'EncryptionTypeMismatch')
|
|
181
186
|
End = Shapes::IntegerShape.new(name: 'End')
|
|
182
187
|
EndEvent = Shapes::StructureShape.new(name: 'EndEvent')
|
|
@@ -208,6 +213,8 @@ module Aws::S3
|
|
|
208
213
|
FilterRuleList = Shapes::ListShape.new(name: 'FilterRuleList', flattened: true)
|
|
209
214
|
FilterRuleName = Shapes::StringShape.new(name: 'FilterRuleName')
|
|
210
215
|
FilterRuleValue = Shapes::StringShape.new(name: 'FilterRuleValue')
|
|
216
|
+
GetBucketAbacOutput = Shapes::StructureShape.new(name: 'GetBucketAbacOutput')
|
|
217
|
+
GetBucketAbacRequest = Shapes::StructureShape.new(name: 'GetBucketAbacRequest')
|
|
211
218
|
GetBucketAccelerateConfigurationOutput = Shapes::StructureShape.new(name: 'GetBucketAccelerateConfigurationOutput')
|
|
212
219
|
GetBucketAccelerateConfigurationRequest = Shapes::StructureShape.new(name: 'GetBucketAccelerateConfigurationRequest')
|
|
213
220
|
GetBucketAclOutput = Shapes::StructureShape.new(name: 'GetBucketAclOutput')
|
|
@@ -497,6 +504,7 @@ module Aws::S3
|
|
|
497
504
|
ProgressEvent = Shapes::StructureShape.new(name: 'ProgressEvent')
|
|
498
505
|
Protocol = Shapes::StringShape.new(name: 'Protocol')
|
|
499
506
|
PublicAccessBlockConfiguration = Shapes::StructureShape.new(name: 'PublicAccessBlockConfiguration')
|
|
507
|
+
PutBucketAbacRequest = Shapes::StructureShape.new(name: 'PutBucketAbacRequest')
|
|
500
508
|
PutBucketAccelerateConfigurationRequest = Shapes::StructureShape.new(name: 'PutBucketAccelerateConfigurationRequest')
|
|
501
509
|
PutBucketAclRequest = Shapes::StructureShape.new(name: 'PutBucketAclRequest')
|
|
502
510
|
PutBucketAnalyticsConfigurationRequest = Shapes::StructureShape.new(name: 'PutBucketAnalyticsConfigurationRequest')
|
|
@@ -686,6 +694,9 @@ module Aws::S3
|
|
|
686
694
|
WriteOffsetBytes = Shapes::IntegerShape.new(name: 'WriteOffsetBytes')
|
|
687
695
|
Years = Shapes::IntegerShape.new(name: 'Years')
|
|
688
696
|
|
|
697
|
+
AbacStatus.add_member(:status, Shapes::ShapeRef.new(shape: BucketAbacStatus, location_name: "Status"))
|
|
698
|
+
AbacStatus.struct_class = Types::AbacStatus
|
|
699
|
+
|
|
689
700
|
AbortIncompleteMultipartUpload.add_member(:days_after_initiation, Shapes::ShapeRef.new(shape: DaysAfterInitiation, location_name: "DaysAfterInitiation"))
|
|
690
701
|
AbortIncompleteMultipartUpload.struct_class = Types::AbortIncompleteMultipartUpload
|
|
691
702
|
|
|
@@ -741,6 +752,9 @@ module Aws::S3
|
|
|
741
752
|
AnalyticsS3BucketDestination.add_member(:prefix, Shapes::ShapeRef.new(shape: Prefix, location_name: "Prefix"))
|
|
742
753
|
AnalyticsS3BucketDestination.struct_class = Types::AnalyticsS3BucketDestination
|
|
743
754
|
|
|
755
|
+
BlockedEncryptionTypes.add_member(:encryption_type, Shapes::ShapeRef.new(shape: EncryptionTypeList, location_name: "EncryptionType"))
|
|
756
|
+
BlockedEncryptionTypes.struct_class = Types::BlockedEncryptionTypes
|
|
757
|
+
|
|
744
758
|
Bucket.add_member(:name, Shapes::ShapeRef.new(shape: BucketName, location_name: "Name"))
|
|
745
759
|
Bucket.add_member(:creation_date, Shapes::ShapeRef.new(shape: CreationDate, location_name: "CreationDate"))
|
|
746
760
|
Bucket.add_member(:bucket_region, Shapes::ShapeRef.new(shape: BucketRegion, location_name: "BucketRegion"))
|
|
@@ -907,6 +921,8 @@ module Aws::S3
|
|
|
907
921
|
CopyObjectRequest.add_member(:grant_read, Shapes::ShapeRef.new(shape: GrantRead, location: "header", location_name: "x-amz-grant-read"))
|
|
908
922
|
CopyObjectRequest.add_member(:grant_read_acp, Shapes::ShapeRef.new(shape: GrantReadACP, location: "header", location_name: "x-amz-grant-read-acp"))
|
|
909
923
|
CopyObjectRequest.add_member(:grant_write_acp, Shapes::ShapeRef.new(shape: GrantWriteACP, location: "header", location_name: "x-amz-grant-write-acp"))
|
|
924
|
+
CopyObjectRequest.add_member(:if_match, Shapes::ShapeRef.new(shape: IfMatch, location: "header", location_name: "If-Match"))
|
|
925
|
+
CopyObjectRequest.add_member(:if_none_match, Shapes::ShapeRef.new(shape: IfNoneMatch, location: "header", location_name: "If-None-Match"))
|
|
910
926
|
CopyObjectRequest.add_member(:key, Shapes::ShapeRef.new(shape: ObjectKey, required: true, location: "uri", location_name: "Key", metadata: {"contextParam" => {"name" => "Key"}}))
|
|
911
927
|
CopyObjectRequest.add_member(:metadata, Shapes::ShapeRef.new(shape: Metadata, location: "headers", location_name: "x-amz-meta-"))
|
|
912
928
|
CopyObjectRequest.add_member(:metadata_directive, Shapes::ShapeRef.new(shape: MetadataDirective, location: "header", location_name: "x-amz-metadata-directive"))
|
|
@@ -1218,6 +1234,8 @@ module Aws::S3
|
|
|
1218
1234
|
EncryptionConfiguration.add_member(:replica_kms_key_id, Shapes::ShapeRef.new(shape: ReplicaKmsKeyID, location_name: "ReplicaKmsKeyID"))
|
|
1219
1235
|
EncryptionConfiguration.struct_class = Types::EncryptionConfiguration
|
|
1220
1236
|
|
|
1237
|
+
EncryptionTypeList.member = Shapes::ShapeRef.new(shape: EncryptionType, location_name: "EncryptionType")
|
|
1238
|
+
|
|
1221
1239
|
EncryptionTypeMismatch.struct_class = Types::EncryptionTypeMismatch
|
|
1222
1240
|
|
|
1223
1241
|
EndEvent.struct_class = Types::EndEvent
|
|
@@ -1252,6 +1270,15 @@ module Aws::S3
|
|
|
1252
1270
|
|
|
1253
1271
|
FilterRuleList.member = Shapes::ShapeRef.new(shape: FilterRule)
|
|
1254
1272
|
|
|
1273
|
+
GetBucketAbacOutput.add_member(:abac_status, Shapes::ShapeRef.new(shape: AbacStatus, location_name: "AbacStatus"))
|
|
1274
|
+
GetBucketAbacOutput.struct_class = Types::GetBucketAbacOutput
|
|
1275
|
+
GetBucketAbacOutput[:payload] = :abac_status
|
|
1276
|
+
GetBucketAbacOutput[:payload_member] = GetBucketAbacOutput.member(:abac_status)
|
|
1277
|
+
|
|
1278
|
+
GetBucketAbacRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
|
|
1279
|
+
GetBucketAbacRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
|
|
1280
|
+
GetBucketAbacRequest.struct_class = Types::GetBucketAbacRequest
|
|
1281
|
+
|
|
1255
1282
|
GetBucketAccelerateConfigurationOutput.add_member(:status, Shapes::ShapeRef.new(shape: BucketAccelerateStatus, location_name: "Status"))
|
|
1256
1283
|
GetBucketAccelerateConfigurationOutput.add_member(:request_charged, Shapes::ShapeRef.new(shape: RequestCharged, location: "header", location_name: "x-amz-request-charged"))
|
|
1257
1284
|
GetBucketAccelerateConfigurationOutput.struct_class = Types::GetBucketAccelerateConfigurationOutput
|
|
@@ -2292,6 +2319,15 @@ module Aws::S3
|
|
|
2292
2319
|
PublicAccessBlockConfiguration.add_member(:restrict_public_buckets, Shapes::ShapeRef.new(shape: Setting, location_name: "RestrictPublicBuckets"))
|
|
2293
2320
|
PublicAccessBlockConfiguration.struct_class = Types::PublicAccessBlockConfiguration
|
|
2294
2321
|
|
|
2322
|
+
PutBucketAbacRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
|
|
2323
|
+
PutBucketAbacRequest.add_member(:content_md5, Shapes::ShapeRef.new(shape: ContentMD5, location: "header", location_name: "Content-MD5"))
|
|
2324
|
+
PutBucketAbacRequest.add_member(:checksum_algorithm, Shapes::ShapeRef.new(shape: ChecksumAlgorithm, location: "header", location_name: "x-amz-sdk-checksum-algorithm"))
|
|
2325
|
+
PutBucketAbacRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
|
|
2326
|
+
PutBucketAbacRequest.add_member(:abac_status, Shapes::ShapeRef.new(shape: AbacStatus, required: true, location_name: "AbacStatus", metadata: {"xmlNamespace" => {"uri" => "http://s3.amazonaws.com/doc/2006-03-01/"}}))
|
|
2327
|
+
PutBucketAbacRequest.struct_class = Types::PutBucketAbacRequest
|
|
2328
|
+
PutBucketAbacRequest[:payload] = :abac_status
|
|
2329
|
+
PutBucketAbacRequest[:payload_member] = PutBucketAbacRequest.member(:abac_status)
|
|
2330
|
+
|
|
2295
2331
|
PutBucketAccelerateConfigurationRequest.add_member(:bucket, Shapes::ShapeRef.new(shape: BucketName, required: true, location: "uri", location_name: "Bucket", metadata: {"contextParam" => {"name" => "Bucket"}}))
|
|
2296
2332
|
PutBucketAccelerateConfigurationRequest.add_member(:accelerate_configuration, Shapes::ShapeRef.new(shape: AccelerateConfiguration, required: true, location_name: "AccelerateConfiguration", metadata: {"xmlNamespace" => {"uri" => "http://s3.amazonaws.com/doc/2006-03-01/"}}))
|
|
2297
2333
|
PutBucketAccelerateConfigurationRequest.add_member(:expected_bucket_owner, Shapes::ShapeRef.new(shape: AccountId, location: "header", location_name: "x-amz-expected-bucket-owner"))
|
|
@@ -2841,6 +2877,7 @@ module Aws::S3
|
|
|
2841
2877
|
|
|
2842
2878
|
ServerSideEncryptionRule.add_member(:apply_server_side_encryption_by_default, Shapes::ShapeRef.new(shape: ServerSideEncryptionByDefault, location_name: "ApplyServerSideEncryptionByDefault"))
|
|
2843
2879
|
ServerSideEncryptionRule.add_member(:bucket_key_enabled, Shapes::ShapeRef.new(shape: BucketKeyEnabled, location_name: "BucketKeyEnabled"))
|
|
2880
|
+
ServerSideEncryptionRule.add_member(:blocked_encryption_types, Shapes::ShapeRef.new(shape: BlockedEncryptionTypes, location_name: "BlockedEncryptionTypes"))
|
|
2844
2881
|
ServerSideEncryptionRule.struct_class = Types::ServerSideEncryptionRule
|
|
2845
2882
|
|
|
2846
2883
|
ServerSideEncryptionRules.member = Shapes::ShapeRef.new(shape: ServerSideEncryptionRule)
|
|
@@ -3332,6 +3369,14 @@ module Aws::S3
|
|
|
3332
3369
|
o.output = Shapes::ShapeRef.new(shape: Shapes::StructureShape.new(struct_class: Aws::EmptyStructure))
|
|
3333
3370
|
end)
|
|
3334
3371
|
|
|
3372
|
+
api.add_operation(:get_bucket_abac, Seahorse::Model::Operation.new.tap do |o|
|
|
3373
|
+
o.name = "GetBucketAbac"
|
|
3374
|
+
o.http_method = "GET"
|
|
3375
|
+
o.http_request_uri = "/?abac"
|
|
3376
|
+
o.input = Shapes::ShapeRef.new(shape: GetBucketAbacRequest)
|
|
3377
|
+
o.output = Shapes::ShapeRef.new(shape: GetBucketAbacOutput)
|
|
3378
|
+
end)
|
|
3379
|
+
|
|
3335
3380
|
api.add_operation(:get_bucket_accelerate_configuration, Seahorse::Model::Operation.new.tap do |o|
|
|
3336
3381
|
o.name = "GetBucketAccelerateConfiguration"
|
|
3337
3382
|
o.http_method = "GET"
|
|
@@ -3766,6 +3811,22 @@ module Aws::S3
|
|
|
3766
3811
|
)
|
|
3767
3812
|
end)
|
|
3768
3813
|
|
|
3814
|
+
api.add_operation(:put_bucket_abac, Seahorse::Model::Operation.new.tap do |o|
|
|
3815
|
+
o.name = "PutBucketAbac"
|
|
3816
|
+
o.http_method = "PUT"
|
|
3817
|
+
o.http_request_uri = "/?abac"
|
|
3818
|
+
o.http_checksum = {
|
|
3819
|
+
"requestAlgorithmMember" => "checksum_algorithm",
|
|
3820
|
+
"requestChecksumRequired" => false,
|
|
3821
|
+
}
|
|
3822
|
+
o.http_checksum = {
|
|
3823
|
+
"requestAlgorithmMember" => "checksum_algorithm",
|
|
3824
|
+
"requestChecksumRequired" => false,
|
|
3825
|
+
}
|
|
3826
|
+
o.input = Shapes::ShapeRef.new(shape: PutBucketAbacRequest)
|
|
3827
|
+
o.output = Shapes::ShapeRef.new(shape: Shapes::StructureShape.new(struct_class: Aws::EmptyStructure))
|
|
3828
|
+
end)
|
|
3829
|
+
|
|
3769
3830
|
api.add_operation(:put_bucket_accelerate_configuration, Seahorse::Model::Operation.new.tap do |o|
|
|
3770
3831
|
o.name = "PutBucketAccelerateConfiguration"
|
|
3771
3832
|
o.http_method = "PUT"
|
|
@@ -358,8 +358,8 @@ module Aws
|
|
|
358
358
|
# {Client#complete_multipart_upload},
|
|
359
359
|
# and {Client#upload_part} can be provided.
|
|
360
360
|
#
|
|
361
|
-
# @option options [Integer] :thread_count (10) The number of parallel
|
|
362
|
-
#
|
|
361
|
+
# @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
|
|
362
|
+
# An additional thread is used internally for task coordination.
|
|
363
363
|
#
|
|
364
364
|
# @option options [Boolean] :tempfile (false) Normally read data is stored
|
|
365
365
|
# in memory when building the parts in order to complete the underlying
|
|
@@ -383,19 +383,18 @@ module Aws
|
|
|
383
383
|
# @see Client#complete_multipart_upload
|
|
384
384
|
# @see Client#upload_part
|
|
385
385
|
def upload_stream(options = {}, &block)
|
|
386
|
-
|
|
386
|
+
upload_opts = options.merge(bucket: bucket_name, key: key)
|
|
387
|
+
executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
|
|
387
388
|
uploader = MultipartStreamUploader.new(
|
|
388
389
|
client: client,
|
|
389
|
-
|
|
390
|
-
tempfile:
|
|
391
|
-
part_size:
|
|
390
|
+
executor: executor,
|
|
391
|
+
tempfile: upload_opts.delete(:tempfile),
|
|
392
|
+
part_size: upload_opts.delete(:part_size)
|
|
392
393
|
)
|
|
393
394
|
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
394
|
-
uploader.upload(
|
|
395
|
-
uploading_options.merge(bucket: bucket_name, key: key),
|
|
396
|
-
&block
|
|
397
|
-
)
|
|
395
|
+
uploader.upload(upload_opts, &block)
|
|
398
396
|
end
|
|
397
|
+
executor.shutdown
|
|
399
398
|
true
|
|
400
399
|
end
|
|
401
400
|
deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
|
|
@@ -458,12 +457,18 @@ module Aws
|
|
|
458
457
|
# @see Client#complete_multipart_upload
|
|
459
458
|
# @see Client#upload_part
|
|
460
459
|
def upload_file(source, options = {})
|
|
461
|
-
|
|
462
|
-
|
|
460
|
+
upload_opts = options.merge(bucket: bucket_name, key: key)
|
|
461
|
+
executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
|
|
462
|
+
uploader = FileUploader.new(
|
|
463
|
+
client: client,
|
|
464
|
+
executor: executor,
|
|
465
|
+
multipart_threshold: upload_opts.delete(:multipart_threshold)
|
|
466
|
+
)
|
|
463
467
|
response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
464
|
-
uploader.upload(source,
|
|
468
|
+
uploader.upload(source, upload_opts)
|
|
465
469
|
end
|
|
466
470
|
yield response if block_given?
|
|
471
|
+
executor.shutdown
|
|
467
472
|
true
|
|
468
473
|
end
|
|
469
474
|
deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
|
|
@@ -512,15 +517,10 @@ module Aws
|
|
|
512
517
|
#
|
|
513
518
|
# @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
|
|
514
519
|
#
|
|
515
|
-
# @option options [String] :version_id The object version id used to retrieve the object.
|
|
516
|
-
#
|
|
517
|
-
# @see https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html ObjectVersioning
|
|
518
|
-
#
|
|
519
520
|
# @option options [String] :checksum_mode ("ENABLED")
|
|
520
|
-
#
|
|
521
|
-
#
|
|
522
|
-
#
|
|
523
|
-
# To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
|
|
521
|
+
# This option is deprecated. Use `:response_checksum_validation` on your S3 client instead.
|
|
522
|
+
# To disable checksum validation, set `response_checksum_validation: 'when_required'`
|
|
523
|
+
# when creating your S3 client.
|
|
524
524
|
#
|
|
525
525
|
# @option options [Callable] :on_checksum_validated
|
|
526
526
|
# Called each time a request's checksum is validated with the checksum algorithm and the
|
|
@@ -539,10 +539,13 @@ module Aws
|
|
|
539
539
|
# @see Client#get_object
|
|
540
540
|
# @see Client#head_object
|
|
541
541
|
def download_file(destination, options = {})
|
|
542
|
-
|
|
542
|
+
download_opts = options.merge(bucket: bucket_name, key: key)
|
|
543
|
+
executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
|
|
544
|
+
downloader = FileDownloader.new(client: client, executor: executor)
|
|
543
545
|
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
544
|
-
downloader.download(destination,
|
|
546
|
+
downloader.download(destination, download_opts)
|
|
545
547
|
end
|
|
548
|
+
executor.shutdown
|
|
546
549
|
true
|
|
547
550
|
end
|
|
548
551
|
deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
|
|
@@ -6,7 +6,9 @@ module Aws
|
|
|
6
6
|
autoload :BucketRegionCache, 'aws-sdk-s3/bucket_region_cache'
|
|
7
7
|
autoload :Encryption, 'aws-sdk-s3/encryption'
|
|
8
8
|
autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
|
|
9
|
+
autoload :EncryptionV3, 'aws-sdk-s3/encryption_v3'
|
|
9
10
|
autoload :FilePart, 'aws-sdk-s3/file_part'
|
|
11
|
+
autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
|
|
10
12
|
autoload :FileUploader, 'aws-sdk-s3/file_uploader'
|
|
11
13
|
autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
|
|
12
14
|
autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aws
|
|
4
|
+
module S3
|
|
5
|
+
# @api private
|
|
6
|
+
class DefaultExecutor
|
|
7
|
+
DEFAULT_MAX_THREADS = 10
|
|
8
|
+
RUNNING = :running
|
|
9
|
+
SHUTTING_DOWN = :shutting_down
|
|
10
|
+
SHUTDOWN = :shutdown
|
|
11
|
+
|
|
12
|
+
def initialize(options = {})
|
|
13
|
+
@max_threads = options[:max_threads] || DEFAULT_MAX_THREADS
|
|
14
|
+
@state = RUNNING
|
|
15
|
+
@queue = Queue.new
|
|
16
|
+
@pool = []
|
|
17
|
+
@mutex = Mutex.new
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Submits a task for execution.
|
|
21
|
+
# @param [Object] args Variable number of arguments to pass to the block
|
|
22
|
+
# @param [Proc] block The block to be executed
|
|
23
|
+
# @return [Boolean] Returns true if the task was submitted successfully
|
|
24
|
+
def post(*args, &block)
|
|
25
|
+
@mutex.synchronize do
|
|
26
|
+
raise 'Executor has been shutdown and is no longer accepting tasks' unless @state == RUNNING
|
|
27
|
+
|
|
28
|
+
@queue << [args, block]
|
|
29
|
+
ensure_worker_available
|
|
30
|
+
end
|
|
31
|
+
true
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Immediately terminates all worker threads and clears pending tasks.
|
|
35
|
+
# This is a forceful shutdown that doesn't wait for running tasks to complete.
|
|
36
|
+
#
|
|
37
|
+
# @return [Boolean] true when termination is complete
|
|
38
|
+
def kill
|
|
39
|
+
@mutex.synchronize do
|
|
40
|
+
@state = SHUTDOWN
|
|
41
|
+
@pool.each(&:kill)
|
|
42
|
+
@pool.clear
|
|
43
|
+
@queue.clear
|
|
44
|
+
end
|
|
45
|
+
true
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Gracefully shuts down the executor, optionally with a timeout.
|
|
49
|
+
# Stops accepting new tasks and waits for running tasks to complete.
|
|
50
|
+
#
|
|
51
|
+
# @param timeout [Numeric, nil] Maximum time in seconds to wait for shutdown.
|
|
52
|
+
# If nil, waits indefinitely. If timeout expires, remaining threads are killed.
|
|
53
|
+
# @return [Boolean] true when shutdown is complete
|
|
54
|
+
def shutdown(timeout = nil)
|
|
55
|
+
@mutex.synchronize do
|
|
56
|
+
return true if @state == SHUTDOWN
|
|
57
|
+
|
|
58
|
+
@state = SHUTTING_DOWN
|
|
59
|
+
@pool.size.times { @queue << :shutdown }
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
if timeout
|
|
63
|
+
deadline = Time.now + timeout
|
|
64
|
+
@pool.each do |thread|
|
|
65
|
+
remaining = deadline - Time.now
|
|
66
|
+
break if remaining <= 0
|
|
67
|
+
|
|
68
|
+
thread.join([remaining, 0].max)
|
|
69
|
+
end
|
|
70
|
+
@pool.select(&:alive?).each(&:kill)
|
|
71
|
+
else
|
|
72
|
+
@pool.each(&:join)
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
@mutex.synchronize do
|
|
76
|
+
@pool.clear
|
|
77
|
+
@state = SHUTDOWN
|
|
78
|
+
end
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
private
|
|
83
|
+
|
|
84
|
+
def ensure_worker_available
|
|
85
|
+
return unless @state == RUNNING
|
|
86
|
+
|
|
87
|
+
@pool.select!(&:alive?)
|
|
88
|
+
@pool << spawn_worker if @pool.size < @max_threads
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def spawn_worker
|
|
92
|
+
Thread.new do
|
|
93
|
+
while (job = @queue.shift)
|
|
94
|
+
break if job == :shutdown
|
|
95
|
+
|
|
96
|
+
args, block = job
|
|
97
|
+
block.call(*args)
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
@@ -6,9 +6,9 @@ module Aws
|
|
|
6
6
|
module S3
|
|
7
7
|
|
|
8
8
|
# [MAINTENANCE MODE] There is a new version of the Encryption Client.
|
|
9
|
-
# AWS strongly recommends upgrading to the {Aws::S3::
|
|
9
|
+
# AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
|
|
10
10
|
# which provides updated data security best practices.
|
|
11
|
-
# See documentation for {Aws::S3::
|
|
11
|
+
# See documentation for {Aws::S3::EncryptionV3::Client}.
|
|
12
12
|
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
13
13
|
# storing the encrypted data in Amazon S3.
|
|
14
14
|
#
|
|
@@ -16,6 +16,8 @@ module Aws
|
|
|
16
16
|
# envelope and encryption cipher.
|
|
17
17
|
def encryption_cipher
|
|
18
18
|
cipher = Utils.aes_encryption_cipher(:CBC)
|
|
19
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
|
|
20
|
+
##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
|
|
19
21
|
envelope = {
|
|
20
22
|
'x-amz-key' => encode64(encrypt(envelope_key(cipher))),
|
|
21
23
|
'x-amz-iv' => encode64(envelope_iv(cipher)),
|
|
@@ -38,6 +38,8 @@ module Aws
|
|
|
38
38
|
io = StringIO.new(io) if String === io
|
|
39
39
|
context.params[:body] = IOEncrypter.new(cipher, io)
|
|
40
40
|
context.params[:metadata] ||= {}
|
|
41
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#content-metadata-mapkeys
|
|
42
|
+
##% - The mapkey "x-amz-unencrypted-content-length" SHOULD be present for V1 format objects.
|
|
41
43
|
context.params[:metadata]['x-amz-unencrypted-content-length'] = io.size
|
|
42
44
|
if context.params.delete(:content_md5)
|
|
43
45
|
warn('Setting content_md5 on client side encrypted objects is deprecated')
|
|
@@ -26,6 +26,8 @@ module Aws
|
|
|
26
26
|
end
|
|
27
27
|
cipher = Utils.aes_encryption_cipher(:CBC)
|
|
28
28
|
cipher.key = key_data.plaintext
|
|
29
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#algorithm-suite-and-message-format-version-compatibility
|
|
30
|
+
##% Objects encrypted with ALG_AES_256_CBC_IV16_NO_KDF MAY use either the V1 or V2 message format version.
|
|
29
31
|
envelope = {
|
|
30
32
|
'x-amz-key-v2' => encode64(key_data.ciphertext_blob),
|
|
31
33
|
'x-amz-iv' => encode64(cipher.iv = cipher.random_iv),
|
|
@@ -5,9 +5,17 @@ require 'forwardable'
|
|
|
5
5
|
module Aws
|
|
6
6
|
module S3
|
|
7
7
|
|
|
8
|
-
REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile]
|
|
9
|
-
SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy]
|
|
8
|
+
REQUIRED_PARAMS = [:key_wrap_schema, :content_encryption_schema, :security_profile].freeze
|
|
9
|
+
SUPPORTED_SECURITY_PROFILES = [:v2, :v2_and_legacy].freeze
|
|
10
|
+
SUPPORTED_COMMITMENT_POLICIES = [:forbid_encrypt_allow_decrypt].freeze
|
|
10
11
|
|
|
12
|
+
# [MAINTENANCE MODE] There is a new version of the Encryption Client.
|
|
13
|
+
# AWS strongly recommends upgrading to the {Aws::S3::EncryptionV3::Client},
|
|
14
|
+
# which provides updated data security best practices.
|
|
15
|
+
# For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
|
|
16
|
+
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
17
|
+
# storing the encrypted data in Amazon S3.
|
|
18
|
+
#
|
|
11
19
|
# Provides an encryption client that encrypts and decrypts data client-side,
|
|
12
20
|
# storing the encrypted data in Amazon S3. The `EncryptionV2::Client` (V2 Client)
|
|
13
21
|
# provides improved security over the `Encryption::Client` (V1 Client)
|
|
@@ -307,15 +315,29 @@ module Aws
|
|
|
307
315
|
# @option options [KMS::Client] :kms_client A default {KMS::Client}
|
|
308
316
|
# is constructed when using KMS to manage encryption keys.
|
|
309
317
|
#
|
|
318
|
+
# @option options [Symbol] :commitment_policy (nil)
|
|
319
|
+
# Optional parameter for migration from V2 to V3. When set to
|
|
320
|
+
# :forbid_encrypt_allow_decrypt, this explicitly indicates you are
|
|
321
|
+
# maintaining V2 encryption behavior while preparing for migration.
|
|
322
|
+
# This allows the V2 client to decrypt V3-encrypted objects while
|
|
323
|
+
# continuing to encrypt new objects using V2 algorithms.
|
|
324
|
+
# Only :forbid_encrypt_allow_decrypt is supported.
|
|
325
|
+
# For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html
|
|
326
|
+
#
|
|
310
327
|
def initialize(options = {})
|
|
311
328
|
validate_params(options)
|
|
312
329
|
@client = extract_client(options)
|
|
313
|
-
@cipher_provider =
|
|
330
|
+
@cipher_provider = build_cipher_provider(options)
|
|
331
|
+
@key_provider = @cipher_provider.key_provider if @cipher_provider.is_a?(DefaultCipherProvider)
|
|
314
332
|
@envelope_location = extract_location(options)
|
|
315
333
|
@instruction_file_suffix = extract_suffix(options)
|
|
316
334
|
@kms_allow_decrypt_with_any_cmk =
|
|
317
335
|
options[:kms_key_id] == :kms_allow_decrypt_with_any_cmk
|
|
318
336
|
@security_profile = extract_security_profile(options)
|
|
337
|
+
@commitment_policy = extract_commitment_policy(options)
|
|
338
|
+
# The v3 cipher is only used for decrypt.
|
|
339
|
+
# Therefore any configured v2 `content_encryption_schema` is going to be incorrect.
|
|
340
|
+
@v3_cipher_provider = build_v3_cipher_provider_for_decrypt(options.reject { |k, _| k == :content_encryption_schema })
|
|
319
341
|
end
|
|
320
342
|
|
|
321
343
|
# @return [S3::Client]
|
|
@@ -341,6 +363,11 @@ module Aws
|
|
|
341
363
|
# by this string.
|
|
342
364
|
attr_reader :instruction_file_suffix
|
|
343
365
|
|
|
366
|
+
# @return [Symbol, nil] Optional commitment policy for V2 to V3 migration.
|
|
367
|
+
# When set to :forbid_encrypt_allow_decrypt, explicitly indicates
|
|
368
|
+
# maintaining V2 encryption behavior while preparing for migration.
|
|
369
|
+
attr_reader :commitment_policy
|
|
370
|
+
|
|
344
371
|
# Uploads an object to Amazon S3, encrypting data client-side.
|
|
345
372
|
# See {S3::Client#put_object} for documentation on accepted
|
|
346
373
|
# request parameters.
|
|
@@ -410,6 +437,7 @@ module Aws
|
|
|
410
437
|
req.handlers.add(DecryptHandler)
|
|
411
438
|
req.context[:encryption] = {
|
|
412
439
|
cipher_provider: @cipher_provider,
|
|
440
|
+
v3_cipher_provider: @v3_cipher_provider,
|
|
413
441
|
envelope_location: envelope_location,
|
|
414
442
|
instruction_file_suffix: instruction_file_suffix,
|
|
415
443
|
kms_encryption_context: kms_encryption_context,
|
|
@@ -423,6 +451,50 @@ module Aws
|
|
|
423
451
|
|
|
424
452
|
private
|
|
425
453
|
|
|
454
|
+
def build_cipher_provider(options)
|
|
455
|
+
if options[:kms_key_id]
|
|
456
|
+
KmsCipherProvider.new(
|
|
457
|
+
kms_key_id: options[:kms_key_id],
|
|
458
|
+
kms_client: kms_client(options),
|
|
459
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
460
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
461
|
+
)
|
|
462
|
+
else
|
|
463
|
+
key_provider = extract_key_provider(options)
|
|
464
|
+
DefaultCipherProvider.new(
|
|
465
|
+
key_provider: key_provider,
|
|
466
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
467
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
468
|
+
)
|
|
469
|
+
end
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def build_v3_cipher_provider_for_decrypt(options)
|
|
473
|
+
if options[:kms_key_id]
|
|
474
|
+
Aws::S3::EncryptionV3::KmsCipherProvider.new(
|
|
475
|
+
kms_key_id: options[:kms_key_id],
|
|
476
|
+
kms_client: kms_client(options),
|
|
477
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
478
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
479
|
+
)
|
|
480
|
+
else
|
|
481
|
+
# Create V3 key provider explicitly for proper namespace consistency
|
|
482
|
+
key_provider = if options[:key_provider]
|
|
483
|
+
options[:key_provider]
|
|
484
|
+
elsif options[:encryption_key]
|
|
485
|
+
Aws::S3::EncryptionV3::DefaultKeyProvider.new(options)
|
|
486
|
+
else
|
|
487
|
+
msg = 'you must pass a :kms_key_id, :key_provider, or :encryption_key'
|
|
488
|
+
raise ArgumentError, msg
|
|
489
|
+
end
|
|
490
|
+
Aws::S3::EncryptionV3::DefaultCipherProvider.new(
|
|
491
|
+
key_provider: key_provider,
|
|
492
|
+
key_wrap_schema: options[:key_wrap_schema],
|
|
493
|
+
content_encryption_schema: options[:content_encryption_schema]
|
|
494
|
+
)
|
|
495
|
+
end
|
|
496
|
+
end
|
|
497
|
+
|
|
426
498
|
# Validate required parameters exist and don't conflict.
|
|
427
499
|
# The cek_alg and wrap_alg are passed on to the CipherProviders
|
|
428
500
|
# and further validated there
|
|
@@ -452,36 +524,19 @@ module Aws
|
|
|
452
524
|
options.delete(:encryption_key)
|
|
453
525
|
options.delete(:envelope_location)
|
|
454
526
|
options.delete(:instruction_file_suffix)
|
|
527
|
+
options.delete(:commitment_policy)
|
|
455
528
|
REQUIRED_PARAMS.each { |p| options.delete(p) }
|
|
456
529
|
S3::Client.new(options)
|
|
457
530
|
end
|
|
458
531
|
end
|
|
459
532
|
|
|
460
533
|
def kms_client(options)
|
|
461
|
-
options[:kms_client] ||
|
|
534
|
+
options[:kms_client] || (@kms_client ||=
|
|
462
535
|
KMS::Client.new(
|
|
463
536
|
region: @client.config.region,
|
|
464
537
|
credentials: @client.config.credentials,
|
|
465
538
|
)
|
|
466
|
-
|
|
467
|
-
end
|
|
468
|
-
|
|
469
|
-
def cipher_provider(options)
|
|
470
|
-
if options[:kms_key_id]
|
|
471
|
-
KmsCipherProvider.new(
|
|
472
|
-
kms_key_id: options[:kms_key_id],
|
|
473
|
-
kms_client: kms_client(options),
|
|
474
|
-
key_wrap_schema: options[:key_wrap_schema],
|
|
475
|
-
content_encryption_schema: options[:content_encryption_schema]
|
|
476
|
-
)
|
|
477
|
-
else
|
|
478
|
-
@key_provider = extract_key_provider(options)
|
|
479
|
-
DefaultCipherProvider.new(
|
|
480
|
-
key_provider: @key_provider,
|
|
481
|
-
key_wrap_schema: options[:key_wrap_schema],
|
|
482
|
-
content_encryption_schema: options[:content_encryption_schema]
|
|
483
|
-
)
|
|
484
|
-
end
|
|
539
|
+
)
|
|
485
540
|
end
|
|
486
541
|
|
|
487
542
|
def extract_key_provider(options)
|
|
@@ -564,7 +619,27 @@ module Aws
|
|
|
564
619
|
end
|
|
565
620
|
security_profile
|
|
566
621
|
end
|
|
622
|
+
|
|
623
|
+
def extract_commitment_policy(options)
|
|
624
|
+
validate_commitment_policy(options[:commitment_policy])
|
|
625
|
+
end
|
|
626
|
+
|
|
627
|
+
def validate_commitment_policy(commitment_policy)
|
|
628
|
+
return nil if commitment_policy.nil?
|
|
629
|
+
|
|
630
|
+
unless SUPPORTED_COMMITMENT_POLICIES.include? commitment_policy
|
|
631
|
+
raise ArgumentError, "Unsupported commitment policy: :#{commitment_policy}. " \
|
|
632
|
+
"The V2 client only supports :forbid_encrypt_allow_decrypt for migration purposes. " \
|
|
633
|
+
"For migration guidance, see: https://docs.aws.amazon.com/sdk-for-ruby/v3/developer-guide/s3-encryption-migration-v2-v3.html"
|
|
634
|
+
end
|
|
635
|
+
commitment_policy
|
|
636
|
+
end
|
|
567
637
|
end
|
|
568
638
|
end
|
|
569
639
|
end
|
|
570
640
|
end
|
|
641
|
+
|
|
642
|
+
##= ../specification/s3-encryption/data-format/content-metadata.md#v1-v2-shared
|
|
643
|
+
##= type=exception
|
|
644
|
+
##= reason=This has never been supported in Ruby
|
|
645
|
+
##% This string MAY be encoded by the esoteric double-encoding scheme used by the S3 web server.
|