aws-sdk-s3 1.114.0 → 1.125.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +94 -0
  3. data/VERSION +1 -1
  4. data/lib/aws-sdk-s3/bucket.rb +51 -28
  5. data/lib/aws-sdk-s3/bucket_acl.rb +9 -3
  6. data/lib/aws-sdk-s3/bucket_cors.rb +12 -4
  7. data/lib/aws-sdk-s3/bucket_lifecycle.rb +12 -4
  8. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +12 -4
  9. data/lib/aws-sdk-s3/bucket_logging.rb +9 -3
  10. data/lib/aws-sdk-s3/bucket_notification.rb +9 -3
  11. data/lib/aws-sdk-s3/bucket_policy.rb +12 -4
  12. data/lib/aws-sdk-s3/bucket_request_payment.rb +9 -3
  13. data/lib/aws-sdk-s3/bucket_tagging.rb +12 -4
  14. data/lib/aws-sdk-s3/bucket_versioning.rb +15 -5
  15. data/lib/aws-sdk-s3/bucket_website.rb +12 -4
  16. data/lib/aws-sdk-s3/client.rb +1721 -1457
  17. data/lib/aws-sdk-s3/client_api.rb +189 -189
  18. data/lib/aws-sdk-s3/customizations/bucket.rb +23 -47
  19. data/lib/aws-sdk-s3/customizations/errors.rb +27 -0
  20. data/lib/aws-sdk-s3/customizations/object.rb +32 -19
  21. data/lib/aws-sdk-s3/customizations/types/permanent_redirect.rb +26 -0
  22. data/lib/aws-sdk-s3/customizations.rb +2 -0
  23. data/lib/aws-sdk-s3/encryption/client.rb +6 -2
  24. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +13 -9
  25. data/lib/aws-sdk-s3/encryptionV2/client.rb +6 -2
  26. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +1 -0
  27. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +10 -6
  28. data/lib/aws-sdk-s3/endpoint_parameters.rb +142 -0
  29. data/lib/aws-sdk-s3/endpoint_provider.rb +733 -0
  30. data/lib/aws-sdk-s3/endpoints.rb +2150 -0
  31. data/lib/aws-sdk-s3/file_downloader.rb +28 -24
  32. data/lib/aws-sdk-s3/file_uploader.rb +8 -6
  33. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +41 -13
  34. data/lib/aws-sdk-s3/multipart_upload.rb +12 -4
  35. data/lib/aws-sdk-s3/multipart_upload_part.rb +10 -4
  36. data/lib/aws-sdk-s3/object.rb +86 -57
  37. data/lib/aws-sdk-s3/object_acl.rb +9 -3
  38. data/lib/aws-sdk-s3/object_copier.rb +7 -5
  39. data/lib/aws-sdk-s3/object_multipart_copier.rb +11 -5
  40. data/lib/aws-sdk-s3/object_summary.rb +72 -46
  41. data/lib/aws-sdk-s3/object_version.rb +21 -9
  42. data/lib/aws-sdk-s3/plugins/accelerate.rb +3 -50
  43. data/lib/aws-sdk-s3/plugins/arn.rb +0 -184
  44. data/lib/aws-sdk-s3/plugins/bucket_dns.rb +3 -39
  45. data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +1 -6
  46. data/lib/aws-sdk-s3/plugins/dualstack.rb +1 -49
  47. data/lib/aws-sdk-s3/plugins/endpoints.rb +262 -0
  48. data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +2 -1
  49. data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +0 -29
  50. data/lib/aws-sdk-s3/plugins/s3_signer.rb +32 -126
  51. data/lib/aws-sdk-s3/presigned_post.rb +9 -16
  52. data/lib/aws-sdk-s3/presigner.rb +20 -33
  53. data/lib/aws-sdk-s3/resource.rb +7 -3
  54. data/lib/aws-sdk-s3/types.rb +406 -3903
  55. data/lib/aws-sdk-s3.rb +5 -1
  56. metadata +10 -9
  57. data/lib/aws-sdk-s3/arn/access_point_arn.rb +0 -69
  58. data/lib/aws-sdk-s3/arn/multi_region_access_point_arn.rb +0 -68
  59. data/lib/aws-sdk-s3/arn/object_lambda_arn.rb +0 -69
  60. data/lib/aws-sdk-s3/arn/outpost_access_point_arn.rb +0 -74
  61. data/lib/aws-sdk-s3/plugins/object_lambda_endpoint.rb +0 -25
@@ -5,28 +5,13 @@ require 'aws-sigv4'
5
5
  module Aws
6
6
  module S3
7
7
  module Plugins
8
- # This plugin is an implementation detail and may be modified.
8
+ # This plugin used to have a V4 signer but it was removed in favor of
9
+ # generic Sign plugin that uses endpoint auth scheme.
10
+ #
9
11
  # @api private
10
12
  class S3Signer < Seahorse::Client::Plugin
11
13
  option(:signature_version, 'v4')
12
14
 
13
- option(:sigv4_signer) do |cfg|
14
- S3Signer.build_v4_signer(
15
- service: 's3',
16
- region: cfg.sigv4_region,
17
- credentials: cfg.credentials
18
- )
19
- end
20
-
21
- option(:sigv4_region) do |cfg|
22
- # S3 removes core's signature_v4 plugin that checks for this
23
- raise Aws::Errors::MissingRegionError if cfg.region.nil?
24
-
25
- Aws::Partitions::EndpointProvider.signing_region(
26
- cfg.region, 's3'
27
- )
28
- end
29
-
30
15
  def add_handlers(handlers, cfg)
31
16
  case cfg.signature_version
32
17
  when 'v4' then add_v4_handlers(handlers)
@@ -39,11 +24,11 @@ module Aws
39
24
 
40
25
  def add_v4_handlers(handlers)
41
26
  handlers.add(CachedBucketRegionHandler, step: :sign, priority: 60)
42
- handlers.add(V4Handler, step: :sign)
43
27
  handlers.add(BucketRegionErrorHandler, step: :sign, priority: 40)
44
28
  end
45
29
 
46
30
  def add_legacy_handler(handlers)
31
+ # generic Sign plugin will be skipped if it sees sigv2
47
32
  handlers.add(LegacyHandler, step: :sign)
48
33
  end
49
34
 
@@ -54,53 +39,6 @@ module Aws
54
39
  end
55
40
  end
56
41
 
57
- class V4Handler < Seahorse::Client::Handler
58
- def call(context)
59
- Aws::Plugins::SignatureV4.apply_signature(
60
- context: context,
61
- signer: sigv4_signer(context)
62
- )
63
- @handler.call(context)
64
- end
65
-
66
- private
67
-
68
- def sigv4_signer(context)
69
- # If the client was configured with the wrong region,
70
- # we have to build a new signer.
71
- if context[:cached_sigv4_region] &&
72
- context[:cached_sigv4_region] != context.config.sigv4_signer.region
73
- S3Signer.build_v4_signer(
74
- service: 's3',
75
- region: context[:cached_sigv4_region],
76
- credentials: context.config.credentials
77
- )
78
- elsif (arn = context.metadata[:s3_arn])
79
- if arn[:arn].is_a?(MultiRegionAccessPointARN)
80
- signing_region = '*'
81
- signing_algorithm = :sigv4a
82
- else
83
- signing_region = arn[:resolved_region]
84
- signing_algorithm = :sigv4
85
- end
86
- S3Signer.build_v4_signer(
87
- service: arn[:arn].service,
88
- signing_algorithm: signing_algorithm,
89
- region: signing_region,
90
- credentials: context.config.credentials
91
- )
92
- elsif context.operation.name == 'WriteGetObjectResponse'
93
- S3Signer.build_v4_signer(
94
- service: 's3-object-lambda',
95
- region: context.config.sigv4_region,
96
- credentials: context.config.credentials
97
- )
98
- else
99
- context.config.sigv4_signer
100
- end
101
- end
102
- end
103
-
104
42
  # This handler will update the http endpoint when the bucket region
105
43
  # is known/cached.
106
44
  class CachedBucketRegionHandler < Seahorse::Client::Handler
@@ -114,11 +52,13 @@ module Aws
114
52
 
115
53
  def check_for_cached_region(context, bucket)
116
54
  cached_region = S3::BUCKET_REGIONS[bucket]
117
- if cached_region && cached_region != context.config.region
55
+ if cached_region &&
56
+ cached_region != context.config.region &&
57
+ !S3Signer.custom_endpoint?(context)
118
58
  context.http_request.endpoint.host = S3Signer.new_hostname(
119
59
  context, cached_region
120
60
  )
121
- context[:cached_sigv4_region] = cached_region
61
+ context[:sigv4_region] = cached_region # Sign plugin will use this
122
62
  end
123
63
  end
124
64
  end
@@ -126,7 +66,8 @@ module Aws
126
66
  # This handler detects when a request fails because of a mismatched bucket
127
67
  # region. It follows up by making a request to determine the correct
128
68
  # region, then finally a version 4 signed request against the correct
129
- # regional endpoint.
69
+ # regional endpoint. This is intended for s3's global endpoint which
70
+ # will return 400 if the bucket is not in region.
130
71
  class BucketRegionErrorHandler < Seahorse::Client::Handler
131
72
  def call(context)
132
73
  response = @handler.call(context)
@@ -138,7 +79,7 @@ module Aws
138
79
  def handle_region_errors(response)
139
80
  if wrong_sigv4_region?(response) &&
140
81
  !fips_region?(response) &&
141
- !custom_endpoint?(response) &&
82
+ !S3Signer.custom_endpoint?(response.context) &&
142
83
  !expired_credentials?(response)
143
84
  get_region_and_retry(response.context)
144
85
  else
@@ -160,25 +101,13 @@ module Aws
160
101
  end
161
102
 
162
103
  def fips_region?(resp)
163
- resp.context.http_request.endpoint.host.include?('fips')
104
+ resp.context.http_request.endpoint.host.include?('s3-fips.')
164
105
  end
165
106
 
166
107
  def expired_credentials?(resp)
167
108
  resp.context.http_response.body_contents.match(/<Code>ExpiredToken<\/Code>/)
168
109
  end
169
110
 
170
- def custom_endpoint?(resp)
171
- resolved_suffix = Aws::Partitions::EndpointProvider.dns_suffix_for(
172
- resp.context.config.region,
173
- 's3',
174
- {
175
- dualstack: resp.context[:use_dualstack_endpoint],
176
- fips: resp.context.config.use_fips_endpoint
177
- }
178
- )
179
- !resp.context.http_request.endpoint.hostname.include?(resolved_suffix)
180
- end
181
-
182
111
  def wrong_sigv4_region?(resp)
183
112
  resp.context.http_response.status_code == 400 &&
184
113
  (resp.context.http_response.headers['x-amz-bucket-region'] ||
@@ -191,18 +120,14 @@ module Aws
191
120
  context, actual_region
192
121
  )
193
122
  context.metadata[:redirect_region] = actual_region
194
- # if it's an ARN, use the service in the ARN
195
- if (arn = context.metadata[:s3_arn])
196
- service = arn[:arn].service
197
- end
198
- Aws::Plugins::SignatureV4.apply_signature(
199
- context: context,
200
- signer: S3Signer.build_v4_signer(
201
- service: service || 's3',
202
- region: actual_region,
203
- credentials: context.config.credentials
204
- )
123
+
124
+ signer = Aws::Plugins::Sign.signer_for(
125
+ context[:auth_scheme],
126
+ context.config,
127
+ actual_region
205
128
  )
129
+
130
+ signer.sign(context)
206
131
  end
207
132
 
208
133
  def region_from_body(body)
@@ -228,41 +153,22 @@ module Aws
228
153
  end
229
154
 
230
155
  class << self
231
- # @option options [required, String] :region
232
- # @option options [required, #credentials] :credentials
233
- # @api private
234
- def build_v4_signer(options = {})
235
- Aws::Sigv4::Signer.new(
236
- service: options[:service],
237
- region: options[:region],
238
- credentials_provider: options[:credentials],
239
- signing_algorithm: options.fetch(:signing_algorithm, :sigv4),
240
- uri_escape_path: false,
241
- unsigned_headers: ['content-length', 'x-amzn-trace-id']
242
- )
156
+ def new_hostname(context, region)
157
+ endpoint_params = context[:endpoint_params].dup
158
+ endpoint_params.region = region
159
+ endpoint_params.endpoint = nil
160
+ endpoint =
161
+ context.config.endpoint_provider.resolve_endpoint(endpoint_params)
162
+ URI(endpoint.url).host
243
163
  end
244
164
 
245
- # Check to see if the bucket is actually an ARN
246
- # Otherwise it will retry with the ARN as the bucket name.
247
- def new_hostname(context, region)
248
- uri = URI.parse(
249
- Aws::Partitions::EndpointProvider.resolve(
250
- region, 's3', 'regional',
251
- {
252
- dualstack: context[:use_dualstack_endpoint],
253
- fips: context.config.use_fips_endpoint
254
- }
255
- )
256
- )
165
+ def custom_endpoint?(context)
166
+ region = context.config.region
167
+ partition = Aws::Endpoints::Matchers.aws_partition(region)
168
+ endpoint = context.http_request.endpoint
257
169
 
258
- if (arn = context.metadata[:s3_arn])
259
- # Retry with the response region and not the ARN resolved one
260
- ARN.resolve_url!(
261
- uri, arn[:arn], region, arn[:fips], arn[:dualstack]
262
- ).host
263
- else
264
- "#{context.params[:bucket]}.#{uri.host}"
265
- end
170
+ !endpoint.hostname.include?(partition['dnsSuffix']) &&
171
+ !endpoint.hostname.include?(partition['dualStackDnsSuffix'])
266
172
  end
267
173
  end
268
174
  end
@@ -629,22 +629,15 @@ module Aws
629
629
  end
630
630
 
631
631
  def bucket_url
632
- url = Aws::Partitions::EndpointProvider.resolve(@bucket_region, 's3')
633
- url = URI.parse(url)
634
- if Plugins::BucketDns.dns_compatible?(@bucket_name, _ssl = true)
635
- if @accelerate
636
- url.host = "#{@bucket_name}.s3-accelerate.amazonaws.com"
637
- else
638
- url.host = "#{@bucket_name}.#{url.host}"
639
- end
640
- else
641
- url.path = "/#{@bucket_name}"
642
- end
643
- if @bucket_region == 'us-east-1'
644
- # keep legacy behavior by default
645
- url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
646
- end
647
- url.to_s
632
+ # Taken from Aws::S3::Endpoints module
633
+ params = Aws::S3::EndpointParameters.new(
634
+ bucket: @bucket_name,
635
+ region: @bucket_region,
636
+ accelerate: @accelerate,
637
+ use_global_endpoint: true
638
+ )
639
+ endpoint = Aws::S3::EndpointProvider.new.resolve_endpoint(params)
640
+ endpoint.url
648
641
  end
649
642
 
650
643
  # @return [Hash]
@@ -133,7 +133,7 @@ module Aws
133
133
  virtual_host = params.delete(:virtual_host)
134
134
  time = params.delete(:time)
135
135
  unsigned_headers = unsigned_headers(params)
136
- scheme = http_scheme(params)
136
+ secure = params.delete(:secure) != false
137
137
  expires_in = expires_in(params)
138
138
 
139
139
  req = @client.build_request(method, params)
@@ -141,7 +141,7 @@ module Aws
141
141
  handle_presigned_url_context(req)
142
142
 
143
143
  x_amz_headers = sign_but_dont_send(
144
- req, expires_in, scheme, time, unsigned_headers, hoist
144
+ req, expires_in, secure, time, unsigned_headers, hoist
145
145
  )
146
146
  [req.send_request.data, x_amz_headers]
147
147
  end
@@ -151,14 +151,6 @@ module Aws
151
151
  BLACKLISTED_HEADERS - whitelist_headers
152
152
  end
153
153
 
154
- def http_scheme(params)
155
- if params.delete(:secure) == false
156
- 'http'
157
- else
158
- @client.config.endpoint.scheme
159
- end
160
- end
161
-
162
154
  def expires_in(params)
163
155
  if (expires_in = params.delete(:expires_in))
164
156
  if expires_in > ONE_WEEK
@@ -175,8 +167,7 @@ module Aws
175
167
  end
176
168
 
177
169
  def use_bucket_as_hostname(req)
178
- req.handlers.remove(Plugins::BucketDns::Handler)
179
- req.handle do |context|
170
+ req.handle(priority: 35) do |context|
180
171
  uri = context.http_request.endpoint
181
172
  uri.host = context.params[:bucket]
182
173
  uri.path.sub!("/#{context.params[:bucket]}", '')
@@ -197,22 +188,21 @@ module Aws
197
188
 
198
189
  # @param [Seahorse::Client::Request] req
199
190
  def sign_but_dont_send(
200
- req, expires_in, scheme, time, unsigned_headers, hoist = true
191
+ req, expires_in, secure, time, unsigned_headers, hoist = true
201
192
  )
202
193
  x_amz_headers = {}
203
194
 
204
195
  http_req = req.context.http_request
205
196
 
206
197
  req.handlers.remove(Aws::S3::Plugins::S3Signer::LegacyHandler)
207
- req.handlers.remove(Aws::S3::Plugins::S3Signer::V4Handler)
198
+ req.handlers.remove(Aws::Plugins::Sign::Handler)
208
199
  req.handlers.remove(Seahorse::Client::Plugins::ContentLength::Handler)
209
200
 
210
201
  req.handle(step: :send) do |context|
211
- if scheme != http_req.endpoint.scheme
212
- endpoint = http_req.endpoint.dup
213
- endpoint.scheme = scheme
214
- endpoint.port = (scheme == 'http' ? 80 : 443)
215
- http_req.endpoint = URI.parse(endpoint.to_s)
202
+ # if an endpoint was not provided, force secure or insecure
203
+ if context.config.regional_endpoint
204
+ http_req.endpoint.scheme = secure ? 'https' : 'http'
205
+ http_req.endpoint.port = secure ? 443 : 80
216
206
  end
217
207
 
218
208
  query = http_req.endpoint.query ? http_req.endpoint.query.split('&') : []
@@ -231,24 +221,21 @@ module Aws
231
221
  end
232
222
  http_req.endpoint.query = query.join('&') unless query.empty?
233
223
 
234
- signing_algorithm = :sigv4
235
-
236
- # If it's an ARN, get the resolved region and service
237
- if (arn = context.metadata[:s3_arn])
238
- region = arn[:resolved_region]
239
- service = arn[:arn].service
240
- region = arn[:arn].is_a?(MultiRegionAccessPointARN) ? '*': arn[:resolved_region]
241
- signing_algorithm = arn[:arn].is_a?(MultiRegionAccessPointARN) ? :sigv4a : :sigv4
242
- end
243
-
224
+ auth_scheme = context[:auth_scheme]
225
+ scheme_name = auth_scheme['name']
226
+ region = if scheme_name == 'sigv4a'
227
+ auth_scheme['signingRegionSet'].first
228
+ else
229
+ auth_scheme['signingRegion']
230
+ end
244
231
  signer = Aws::Sigv4::Signer.new(
245
- service: service || 's3',
232
+ service: auth_scheme['signingName'] || 's3',
246
233
  region: region || context.config.region,
247
- signing_algorithm: signing_algorithm,
248
234
  credentials_provider: context.config.credentials,
235
+ signing_algorithm: scheme_name.to_sym,
236
+ uri_escape_path: !!!auth_scheme['disableDoubleEncoding'],
249
237
  unsigned_headers: unsigned_headers,
250
- apply_checksum_header: false,
251
- uri_escape_path: false
238
+ apply_checksum_header: false
252
239
  )
253
240
 
254
241
  url = signer.presign_url(
@@ -41,7 +41,7 @@ module Aws::S3
41
41
  # acl: "private", # accepts private, public-read, public-read-write, authenticated-read
42
42
  # bucket: "BucketName", # required
43
43
  # create_bucket_configuration: {
44
- # location_constraint: "af-south-1", # accepts af-south-1, ap-east-1, ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-south-1, ap-southeast-1, ap-southeast-2, ca-central-1, cn-north-1, cn-northwest-1, EU, eu-central-1, eu-north-1, eu-south-1, eu-west-1, eu-west-2, eu-west-3, me-south-1, sa-east-1, us-east-2, us-gov-east-1, us-gov-west-1, us-west-1, us-west-2
44
+ # location_constraint: "af-south-1", # accepts af-south-1, ap-east-1, ap-northeast-1, ap-northeast-2, ap-northeast-3, ap-south-1, ap-southeast-1, ap-southeast-2, ap-southeast-3, ca-central-1, cn-north-1, cn-northwest-1, EU, eu-central-1, eu-north-1, eu-south-1, eu-west-1, eu-west-2, eu-west-3, me-south-1, sa-east-1, us-east-2, us-gov-east-1, us-gov-west-1, us-west-1, us-west-2
45
45
  # },
46
46
  # grant_full_control: "GrantFullControl",
47
47
  # grant_read: "GrantRead",
@@ -94,7 +94,9 @@ module Aws::S3
94
94
  # equivalent form of this ACL expressed in the XML format.
95
95
  # @return [Bucket]
96
96
  def create_bucket(options = {})
97
- @client.create_bucket(options)
97
+ Aws::Plugins::UserAgent.feature('resource') do
98
+ @client.create_bucket(options)
99
+ end
98
100
  Bucket.new(
99
101
  name: options[:bucket],
100
102
  client: @client
@@ -120,7 +122,9 @@ module Aws::S3
120
122
  def buckets(options = {})
121
123
  batches = Enumerator.new do |y|
122
124
  batch = []
123
- resp = @client.list_buckets(options)
125
+ resp = Aws::Plugins::UserAgent.feature('resource') do
126
+ @client.list_buckets(options)
127
+ end
124
128
  resp.data.buckets.each do |b|
125
129
  batch << Bucket.new(
126
130
  name: b.name,