aws-sdk-s3 1.127.0 → 1.141.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +87 -0
  3. data/VERSION +1 -1
  4. data/lib/aws-sdk-s3/bucket.rb +421 -81
  5. data/lib/aws-sdk-s3/bucket_acl.rb +9 -9
  6. data/lib/aws-sdk-s3/bucket_cors.rb +12 -12
  7. data/lib/aws-sdk-s3/bucket_lifecycle.rb +12 -12
  8. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +12 -12
  9. data/lib/aws-sdk-s3/bucket_logging.rb +16 -9
  10. data/lib/aws-sdk-s3/bucket_notification.rb +3 -3
  11. data/lib/aws-sdk-s3/bucket_policy.rb +58 -14
  12. data/lib/aws-sdk-s3/bucket_request_payment.rb +9 -9
  13. data/lib/aws-sdk-s3/bucket_tagging.rb +12 -12
  14. data/lib/aws-sdk-s3/bucket_versioning.rb +27 -27
  15. data/lib/aws-sdk-s3/bucket_website.rb +12 -12
  16. data/lib/aws-sdk-s3/client.rb +5707 -2536
  17. data/lib/aws-sdk-s3/client_api.rb +111 -16
  18. data/lib/aws-sdk-s3/customizations/errors.rb +1 -1
  19. data/lib/aws-sdk-s3/customizations/object.rb +63 -0
  20. data/lib/aws-sdk-s3/customizations.rb +5 -0
  21. data/lib/aws-sdk-s3/endpoint_parameters.rb +36 -0
  22. data/lib/aws-sdk-s3/endpoint_provider.rb +104 -246
  23. data/lib/aws-sdk-s3/endpoints.rb +440 -0
  24. data/lib/aws-sdk-s3/express_credentials.rb +55 -0
  25. data/lib/aws-sdk-s3/express_credentials_cache.rb +30 -0
  26. data/lib/aws-sdk-s3/express_credentials_provider.rb +36 -0
  27. data/lib/aws-sdk-s3/file_downloader.rb +142 -21
  28. data/lib/aws-sdk-s3/multipart_file_uploader.rb +0 -1
  29. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +0 -1
  30. data/lib/aws-sdk-s3/multipart_upload.rb +69 -16
  31. data/lib/aws-sdk-s3/multipart_upload_part.rb +160 -35
  32. data/lib/aws-sdk-s3/object.rb +1504 -235
  33. data/lib/aws-sdk-s3/object_acl.rb +29 -15
  34. data/lib/aws-sdk-s3/object_multipart_copier.rb +33 -17
  35. data/lib/aws-sdk-s3/object_summary.rb +1367 -254
  36. data/lib/aws-sdk-s3/object_version.rb +297 -42
  37. data/lib/aws-sdk-s3/plugins/endpoints.rb +13 -2
  38. data/lib/aws-sdk-s3/plugins/express_session_auth.rb +90 -0
  39. data/lib/aws-sdk-s3/plugins/location_constraint.rb +3 -1
  40. data/lib/aws-sdk-s3/plugins/md5s.rb +2 -1
  41. data/lib/aws-sdk-s3/presigned_post.rb +52 -43
  42. data/lib/aws-sdk-s3/presigner.rb +2 -2
  43. data/lib/aws-sdk-s3/resource.rb +83 -11
  44. data/lib/aws-sdk-s3/types.rb +4500 -1351
  45. data/lib/aws-sdk-s3.rb +1 -1
  46. metadata +11 -7
@@ -0,0 +1,55 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'set'
4
+
5
+ module Aws
6
+ module S3
7
+ # @api private
8
+ class ExpressCredentials
9
+ include CredentialProvider
10
+ include RefreshingCredentials
11
+
12
+ SYNC_EXPIRATION_LENGTH = 60 # 1 minute
13
+ ASYNC_EXPIRATION_LENGTH = 120 # 2 minutes
14
+
15
+ def initialize(options = {})
16
+ @client = options[:client]
17
+ @create_session_params = {}
18
+ options.each_pair do |key, value|
19
+ if self.class.create_session_options.include?(key)
20
+ @create_session_params[key] = value
21
+ end
22
+ end
23
+ @async_refresh = true
24
+ super
25
+ end
26
+
27
+ # @return [S3::Client]
28
+ attr_reader :client
29
+
30
+ private
31
+
32
+ def refresh
33
+ c = @client.create_session(@create_session_params).credentials
34
+ @credentials = Credentials.new(
35
+ c.access_key_id,
36
+ c.secret_access_key,
37
+ c.session_token
38
+ )
39
+ @expiration = c.expiration
40
+ end
41
+
42
+ class << self
43
+
44
+ # @api private
45
+ def create_session_options
46
+ @cso ||= begin
47
+ input = S3::Client.api.operation(:create_session).input
48
+ Set.new(input.shape.member_names)
49
+ end
50
+ end
51
+
52
+ end
53
+ end
54
+ end
55
+ end
@@ -0,0 +1,30 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ # @api private
6
+ class ExpressCredentialsCache
7
+ def initialize
8
+ @credentials = {}
9
+ @mutex = Mutex.new
10
+ end
11
+
12
+ def [](bucket_name)
13
+ @mutex.synchronize { @credentials[bucket_name] }
14
+ end
15
+
16
+ def []=(bucket_name, credential_provider)
17
+ @mutex.synchronize do
18
+ @credentials[bucket_name] = credential_provider
19
+ end
20
+ end
21
+
22
+ def clear
23
+ @mutex.synchronize { @credentials = {} }
24
+ end
25
+ end
26
+
27
+ # @api private
28
+ EXPRESS_CREDENTIALS_CACHE = ExpressCredentialsCache.new
29
+ end
30
+ end
@@ -0,0 +1,36 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ # Returns Credentials class for S3 Express. Accepts CreateSession
6
+ # params as options. See {Client#create_session} for details.
7
+ class ExpressCredentialsProvider
8
+ # @param [Hash] options
9
+ # @option options [Client] :client The S3 client used to create the session.
10
+ # @option options [String] :session_mode (see: {Client#create_session})
11
+ # @option options [Callable] :before_refresh Proc called before
12
+ # credentials are refreshed.
13
+ def initialize(options = {})
14
+ @client = options.delete(:client)
15
+ @options = options
16
+ @cache = EXPRESS_CREDENTIALS_CACHE
17
+ end
18
+
19
+ def express_credentials_for(bucket)
20
+ @cache[bucket] || new_credentials_for(bucket)
21
+ end
22
+
23
+ attr_accessor :client
24
+
25
+ private
26
+
27
+ def new_credentials_for(bucket)
28
+ @cache[bucket] = ExpressCredentials.new(
29
+ bucket: bucket,
30
+ client: @client,
31
+ **@options
32
+ )
33
+ end
34
+ end
35
+ end
36
+ end
@@ -32,6 +32,18 @@ module Aws
32
32
  }
33
33
  @params[:version_id] = options[:version_id] if options[:version_id]
34
34
 
35
+ # checksum_mode only supports the value "ENABLED"
36
+ # falsey values (false/nil) or "DISABLED" should be considered
37
+ # disabled and the api parameter should be unset.
38
+ if (checksum_mode = options.fetch(:checksum_mode, 'ENABLED'))
39
+ @params[:checksum_mode] = checksum_mode unless checksum_mode.upcase == 'DISABLED'
40
+ end
41
+ @on_checksum_validated = options[:on_checksum_validated]
42
+
43
+ @progress_callback = options[:progress_callback]
44
+
45
+ validate!
46
+
35
47
  Aws::Plugins::UserAgent.feature('s3-transfer') do
36
48
  case @mode
37
49
  when 'auto' then multipart_download
@@ -39,7 +51,7 @@ module Aws
39
51
  when 'get_range'
40
52
  if @chunk_size
41
53
  resp = @client.head_object(@params)
42
- multithreaded_get_by_ranges(construct_chunks(resp.content_length))
54
+ multithreaded_get_by_ranges(resp.content_length)
43
55
  else
44
56
  msg = 'In :get_range mode, :chunk_size must be provided'
45
57
  raise ArgumentError, msg
@@ -54,6 +66,17 @@ module Aws
54
66
 
55
67
  private
56
68
 
69
+ def validate!
70
+ if @on_checksum_validated && @params[:checksum_mode] != 'ENABLED'
71
+ raise ArgumentError, "You must set checksum_mode: 'ENABLED' " +
72
+ "when providing a on_checksum_validated callback"
73
+ end
74
+
75
+ if @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
76
+ raise ArgumentError, 'on_checksum_validated must be callable'
77
+ end
78
+ end
79
+
57
80
  def multipart_download
58
81
  resp = @client.head_object(@params.merge(part_number: 1))
59
82
  count = resp.parts_count
@@ -61,7 +84,7 @@ module Aws
61
84
  if resp.content_length <= MIN_CHUNK_SIZE
62
85
  single_request
63
86
  else
64
- multithreaded_get_by_ranges(construct_chunks(resp.content_length))
87
+ multithreaded_get_by_ranges(resp.content_length)
65
88
  end
66
89
  else
67
90
  # partNumber is an option
@@ -78,9 +101,9 @@ module Aws
78
101
  chunk_size = compute_chunk(file_size)
79
102
  part_size = (file_size.to_f / count.to_f).ceil
80
103
  if chunk_size < part_size
81
- multithreaded_get_by_ranges(construct_chunks(file_size))
104
+ multithreaded_get_by_ranges(file_size)
82
105
  else
83
- multithreaded_get_by_parts(count)
106
+ multithreaded_get_by_parts(count, file_size)
84
107
  end
85
108
  end
86
109
 
@@ -112,27 +135,64 @@ module Aws
112
135
  chunks.each_slice(@thread_count).to_a
113
136
  end
114
137
 
115
- def multithreaded_get_by_ranges(chunks)
116
- thread_batches(chunks, 'range')
138
+ def multithreaded_get_by_ranges(file_size)
139
+ offset = 0
140
+ default_chunk_size = compute_chunk(file_size)
141
+ chunks = []
142
+ part_number = 1 # parts start at 1
143
+ while offset < file_size
144
+ progress = offset + default_chunk_size
145
+ progress = file_size if progress > file_size
146
+ range = "bytes=#{offset}-#{progress - 1}"
147
+ chunks << Part.new(
148
+ part_number: part_number,
149
+ size: (progress-offset),
150
+ params: @params.merge(range: range)
151
+ )
152
+ part_number += 1
153
+ offset = progress
154
+ end
155
+ download_in_threads(PartList.new(chunks), file_size)
117
156
  end
118
157
 
119
- def multithreaded_get_by_parts(parts)
120
- thread_batches(parts, 'part_number')
158
+ def multithreaded_get_by_parts(n_parts, total_size)
159
+ parts = (1..n_parts).map do |part|
160
+ Part.new(part_number: part, params: @params.merge(part_number: part))
161
+ end
162
+ download_in_threads(PartList.new(parts), total_size)
121
163
  end
122
164
 
123
- def thread_batches(chunks, param)
124
- batches(chunks, param).each do |batch|
125
- threads = []
126
- batch.each do |chunk|
127
- threads << Thread.new do
128
- resp = @client.get_object(
129
- @params.merge(param.to_sym => chunk)
130
- )
131
- write(resp)
165
+ def download_in_threads(pending, total_size)
166
+ threads = []
167
+ if @progress_callback
168
+ progress = MultipartProgress.new(pending, total_size, @progress_callback)
169
+ end
170
+ @thread_count.times do
171
+ thread = Thread.new do
172
+ begin
173
+ while part = pending.shift
174
+ if progress
175
+ part.params[:on_chunk_received] =
176
+ proc do |_chunk, bytes, total|
177
+ progress.call(part.part_number, bytes, total)
178
+ end
179
+ end
180
+ resp = @client.get_object(part.params)
181
+ write(resp)
182
+ if @on_checksum_validated && resp.checksum_validated
183
+ @on_checksum_validated.call(resp.checksum_validated, resp)
184
+ end
185
+ end
186
+ nil
187
+ rescue => error
188
+ # keep other threads from downloading other parts
189
+ pending.clear!
190
+ raise error
132
191
  end
133
192
  end
134
- threads.each(&:join)
193
+ threads << thread
135
194
  end
195
+ threads.map(&:value).compact
136
196
  end
137
197
 
138
198
  def write(resp)
@@ -142,9 +202,70 @@ module Aws
142
202
  end
143
203
 
144
204
  def single_request
145
- @client.get_object(
146
- @params.merge(response_target: @path)
147
- )
205
+ params = @params.merge(response_target: @path)
206
+ params[:on_chunk_received] = single_part_progress if @progress_callback
207
+ resp = @client.get_object(params)
208
+
209
+ return resp unless @on_checksum_validated
210
+
211
+ if resp.checksum_validated
212
+ @on_checksum_validated.call(resp.checksum_validated, resp)
213
+ end
214
+
215
+ resp
216
+ end
217
+
218
+ def single_part_progress
219
+ proc do |_chunk, bytes_read, total_size|
220
+ @progress_callback.call([bytes_read], [total_size], total_size)
221
+ end
222
+ end
223
+
224
+ class Part < Struct.new(:part_number, :size, :params)
225
+ include Aws::Structure
226
+ end
227
+
228
+ # @api private
229
+ class PartList
230
+ include Enumerable
231
+ def initialize(parts = [])
232
+ @parts = parts
233
+ @mutex = Mutex.new
234
+ end
235
+
236
+ def shift
237
+ @mutex.synchronize { @parts.shift }
238
+ end
239
+
240
+ def size
241
+ @mutex.synchronize { @parts.size }
242
+ end
243
+
244
+ def clear!
245
+ @mutex.synchronize { @parts.clear }
246
+ end
247
+
248
+ def each(&block)
249
+ @mutex.synchronize { @parts.each(&block) }
250
+ end
251
+ end
252
+
253
+ # @api private
254
+ class MultipartProgress
255
+ def initialize(parts, total_size, progress_callback)
256
+ @bytes_received = Array.new(parts.size, 0)
257
+ @part_sizes = parts.map(&:size)
258
+ @total_size = total_size
259
+ @progress_callback = progress_callback
260
+ end
261
+
262
+ def call(part_number, bytes_received, total)
263
+ # part numbers start at 1
264
+ @bytes_received[part_number - 1] = bytes_received
265
+ # part size may not be known until we get the first response
266
+ @part_sizes[part_number - 1] ||= total
267
+ @progress_callback.call(@bytes_received, @part_sizes, @total_size)
268
+ end
148
269
  end
149
270
  end
150
271
  end
@@ -175,7 +175,6 @@ module Aws
175
175
  error
176
176
  end
177
177
  end
178
- thread.abort_on_exception = true
179
178
  threads << thread
180
179
  end
181
180
  threads.map(&:value).compact
@@ -192,7 +192,6 @@ module Aws
192
192
  error
193
193
  end
194
194
  end
195
- thread.abort_on_exception = true
196
195
  thread
197
196
  end
198
197
  end
@@ -69,6 +69,11 @@ module Aws::S3
69
69
  end
70
70
 
71
71
  # The class of storage used to store the object.
72
+ #
73
+ # <note markdown="1"> **Directory buckets** - Only the S3 Express One Zone storage class is
74
+ # supported by directory buckets to store objects.
75
+ #
76
+ # </note>
72
77
  # @return [String]
73
78
  def storage_class
74
79
  data[:storage_class]
@@ -76,6 +81,11 @@ module Aws::S3
76
81
 
77
82
  # Specifies the owner of the object that is part of the multipart
78
83
  # upload.
84
+ #
85
+ # <note markdown="1"> **Directory buckets** - The bucket owner is returned as the object
86
+ # owner for all the objects.
87
+ #
88
+ # </note>
79
89
  # @return [Types::Owner]
80
90
  def owner
81
91
  data[:owner]
@@ -234,17 +244,23 @@ module Aws::S3
234
244
  # @option options [String] :request_payer
235
245
  # Confirms that the requester knows that they will be charged for the
236
246
  # request. Bucket owners need not specify this parameter in their
237
- # requests. For information about downloading objects from Requester
247
+ # requests. If either the source or destination S3 bucket has Requester
248
+ # Pays enabled, the requester will pay for corresponding charges to copy
249
+ # the object. For information about downloading objects from Requester
238
250
  # Pays buckets, see [Downloading Objects in Requester Pays Buckets][1]
239
251
  # in the *Amazon S3 User Guide*.
240
252
  #
253
+ # <note markdown="1"> This functionality is not supported for directory buckets.
254
+ #
255
+ # </note>
256
+ #
241
257
  #
242
258
  #
243
259
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
244
260
  # @option options [String] :expected_bucket_owner
245
- # The account ID of the expected bucket owner. If the bucket is owned by
246
- # a different account, the request fails with the HTTP status code `403
247
- # Forbidden` (access denied).
261
+ # The account ID of the expected bucket owner. If the account ID that
262
+ # you provide does not match the actual owner of the bucket, the request
263
+ # fails with the HTTP status code `403 Forbidden` (access denied).
248
264
  # @return [Types::AbortMultipartUploadOutput]
249
265
  def abort(options = {})
250
266
  options = options.merge(
@@ -329,32 +345,47 @@ module Aws::S3
329
345
  # @option options [String] :request_payer
330
346
  # Confirms that the requester knows that they will be charged for the
331
347
  # request. Bucket owners need not specify this parameter in their
332
- # requests. For information about downloading objects from Requester
348
+ # requests. If either the source or destination S3 bucket has Requester
349
+ # Pays enabled, the requester will pay for corresponding charges to copy
350
+ # the object. For information about downloading objects from Requester
333
351
  # Pays buckets, see [Downloading Objects in Requester Pays Buckets][1]
334
352
  # in the *Amazon S3 User Guide*.
335
353
  #
354
+ # <note markdown="1"> This functionality is not supported for directory buckets.
355
+ #
356
+ # </note>
357
+ #
336
358
  #
337
359
  #
338
360
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
339
361
  # @option options [String] :expected_bucket_owner
340
- # The account ID of the expected bucket owner. If the bucket is owned by
341
- # a different account, the request fails with the HTTP status code `403
342
- # Forbidden` (access denied).
362
+ # The account ID of the expected bucket owner. If the account ID that
363
+ # you provide does not match the actual owner of the bucket, the request
364
+ # fails with the HTTP status code `403 Forbidden` (access denied).
343
365
  # @option options [String] :sse_customer_algorithm
344
366
  # The server-side encryption (SSE) algorithm used to encrypt the object.
345
- # This parameter is needed only when the object was created using a
346
- # checksum algorithm. For more information, see [Protecting data using
347
- # SSE-C keys][1] in the *Amazon S3 User Guide*.
367
+ # This parameter is required only when the object was created using a
368
+ # checksum algorithm or if your bucket policy requires the use of SSE-C.
369
+ # For more information, see [Protecting data using SSE-C keys][1] in the
370
+ # *Amazon S3 User Guide*.
348
371
  #
372
+ # <note markdown="1"> This functionality is not supported for directory buckets.
349
373
  #
374
+ # </note>
350
375
  #
351
- # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
376
+ #
377
+ #
378
+ # [1]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerSideEncryptionCustomerKeys.html#ssec-require-condition-key
352
379
  # @option options [String] :sse_customer_key
353
380
  # The server-side encryption (SSE) customer managed key. This parameter
354
381
  # is needed only when the object was created using a checksum algorithm.
355
382
  # For more information, see [Protecting data using SSE-C keys][1] in the
356
383
  # *Amazon S3 User Guide*.
357
384
  #
385
+ # <note markdown="1"> This functionality is not supported for directory buckets.
386
+ #
387
+ # </note>
388
+ #
358
389
  #
359
390
  #
360
391
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -364,6 +395,10 @@ module Aws::S3
364
395
  # algorithm. For more information, see [Protecting data using SSE-C
365
396
  # keys][1] in the *Amazon S3 User Guide*.
366
397
  #
398
+ # <note markdown="1"> This functionality is not supported for directory buckets.
399
+ #
400
+ # </note>
401
+ #
367
402
  #
368
403
  #
369
404
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -420,23 +455,33 @@ module Aws::S3
420
455
  # @option options [String] :request_payer
421
456
  # Confirms that the requester knows that they will be charged for the
422
457
  # request. Bucket owners need not specify this parameter in their
423
- # requests. For information about downloading objects from Requester
458
+ # requests. If either the source or destination S3 bucket has Requester
459
+ # Pays enabled, the requester will pay for corresponding charges to copy
460
+ # the object. For information about downloading objects from Requester
424
461
  # Pays buckets, see [Downloading Objects in Requester Pays Buckets][1]
425
462
  # in the *Amazon S3 User Guide*.
426
463
  #
464
+ # <note markdown="1"> This functionality is not supported for directory buckets.
465
+ #
466
+ # </note>
467
+ #
427
468
  #
428
469
  #
429
470
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html
430
471
  # @option options [String] :expected_bucket_owner
431
- # The account ID of the expected bucket owner. If the bucket is owned by
432
- # a different account, the request fails with the HTTP status code `403
433
- # Forbidden` (access denied).
472
+ # The account ID of the expected bucket owner. If the account ID that
473
+ # you provide does not match the actual owner of the bucket, the request
474
+ # fails with the HTTP status code `403 Forbidden` (access denied).
434
475
  # @option options [String] :sse_customer_algorithm
435
476
  # The server-side encryption (SSE) algorithm used to encrypt the object.
436
477
  # This parameter is needed only when the object was created using a
437
478
  # checksum algorithm. For more information, see [Protecting data using
438
479
  # SSE-C keys][1] in the *Amazon S3 User Guide*.
439
480
  #
481
+ # <note markdown="1"> This functionality is not supported for directory buckets.
482
+ #
483
+ # </note>
484
+ #
440
485
  #
441
486
  #
442
487
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -446,6 +491,10 @@ module Aws::S3
446
491
  # For more information, see [Protecting data using SSE-C keys][1] in the
447
492
  # *Amazon S3 User Guide*.
448
493
  #
494
+ # <note markdown="1"> This functionality is not supported for directory buckets.
495
+ #
496
+ # </note>
497
+ #
449
498
  #
450
499
  #
451
500
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
@@ -455,6 +504,10 @@ module Aws::S3
455
504
  # algorithm. For more information, see [Protecting data using SSE-C
456
505
  # keys][1] in the *Amazon S3 User Guide*.
457
506
  #
507
+ # <note markdown="1"> This functionality is not supported for directory buckets.
508
+ #
509
+ # </note>
510
+ #
458
511
  #
459
512
  #
460
513
  # [1]: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html