aws-sdk-s3 1.103.0 → 1.202.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +664 -0
  3. data/VERSION +1 -1
  4. data/lib/aws-sdk-s3/access_grants_credentials.rb +57 -0
  5. data/lib/aws-sdk-s3/access_grants_credentials_provider.rb +250 -0
  6. data/lib/aws-sdk-s3/bucket.rb +858 -116
  7. data/lib/aws-sdk-s3/bucket_acl.rb +32 -9
  8. data/lib/aws-sdk-s3/bucket_cors.rb +38 -13
  9. data/lib/aws-sdk-s3/bucket_lifecycle.rb +43 -12
  10. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +100 -13
  11. data/lib/aws-sdk-s3/bucket_logging.rb +35 -6
  12. data/lib/aws-sdk-s3/bucket_notification.rb +27 -9
  13. data/lib/aws-sdk-s3/bucket_policy.rb +79 -10
  14. data/lib/aws-sdk-s3/bucket_region_cache.rb +9 -5
  15. data/lib/aws-sdk-s3/bucket_request_payment.rb +29 -7
  16. data/lib/aws-sdk-s3/bucket_tagging.rb +35 -11
  17. data/lib/aws-sdk-s3/bucket_versioning.rb +108 -17
  18. data/lib/aws-sdk-s3/bucket_website.rb +35 -11
  19. data/lib/aws-sdk-s3/client.rb +11799 -3636
  20. data/lib/aws-sdk-s3/client_api.rb +1201 -276
  21. data/lib/aws-sdk-s3/customizations/bucket.rb +23 -47
  22. data/lib/aws-sdk-s3/customizations/errors.rb +40 -0
  23. data/lib/aws-sdk-s3/customizations/object.rb +216 -70
  24. data/lib/aws-sdk-s3/customizations/object_summary.rb +5 -0
  25. data/lib/aws-sdk-s3/customizations/object_version.rb +13 -0
  26. data/lib/aws-sdk-s3/customizations/types/permanent_redirect.rb +26 -0
  27. data/lib/aws-sdk-s3/customizations.rb +27 -29
  28. data/lib/aws-sdk-s3/default_executor.rb +103 -0
  29. data/lib/aws-sdk-s3/encryption/client.rb +6 -2
  30. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +13 -9
  31. data/lib/aws-sdk-s3/encryptionV2/client.rb +6 -2
  32. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +1 -0
  33. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +10 -6
  34. data/lib/aws-sdk-s3/endpoint_parameters.rb +181 -0
  35. data/lib/aws-sdk-s3/endpoint_provider.rb +716 -0
  36. data/lib/aws-sdk-s3/endpoints.rb +1518 -0
  37. data/lib/aws-sdk-s3/errors.rb +58 -0
  38. data/lib/aws-sdk-s3/express_credentials.rb +55 -0
  39. data/lib/aws-sdk-s3/express_credentials_provider.rb +59 -0
  40. data/lib/aws-sdk-s3/file_downloader.rb +241 -87
  41. data/lib/aws-sdk-s3/file_uploader.rb +16 -13
  42. data/lib/aws-sdk-s3/legacy_signer.rb +2 -1
  43. data/lib/aws-sdk-s3/multipart_download_error.rb +8 -0
  44. data/lib/aws-sdk-s3/multipart_file_uploader.rb +108 -86
  45. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +110 -92
  46. data/lib/aws-sdk-s3/multipart_upload.rb +294 -19
  47. data/lib/aws-sdk-s3/multipart_upload_error.rb +3 -4
  48. data/lib/aws-sdk-s3/multipart_upload_part.rb +297 -31
  49. data/lib/aws-sdk-s3/object.rb +2224 -269
  50. data/lib/aws-sdk-s3/object_acl.rb +59 -17
  51. data/lib/aws-sdk-s3/object_copier.rb +7 -5
  52. data/lib/aws-sdk-s3/object_multipart_copier.rb +48 -23
  53. data/lib/aws-sdk-s3/object_summary.rb +1915 -220
  54. data/lib/aws-sdk-s3/object_version.rb +450 -58
  55. data/lib/aws-sdk-s3/plugins/accelerate.rb +3 -44
  56. data/lib/aws-sdk-s3/plugins/access_grants.rb +178 -0
  57. data/lib/aws-sdk-s3/plugins/arn.rb +0 -197
  58. data/lib/aws-sdk-s3/plugins/bucket_dns.rb +3 -39
  59. data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +1 -6
  60. data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +31 -0
  61. data/lib/aws-sdk-s3/plugins/dualstack.rb +1 -55
  62. data/lib/aws-sdk-s3/plugins/endpoints.rb +86 -0
  63. data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +2 -1
  64. data/lib/aws-sdk-s3/plugins/express_session_auth.rb +88 -0
  65. data/lib/aws-sdk-s3/plugins/http_200_errors.rb +55 -18
  66. data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +6 -29
  67. data/lib/aws-sdk-s3/plugins/location_constraint.rb +3 -1
  68. data/lib/aws-sdk-s3/plugins/md5s.rb +10 -68
  69. data/lib/aws-sdk-s3/plugins/s3_signer.rb +42 -111
  70. data/lib/aws-sdk-s3/plugins/streaming_retry.rb +28 -9
  71. data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb +2 -1
  72. data/lib/aws-sdk-s3/presigned_post.rb +99 -78
  73. data/lib/aws-sdk-s3/presigner.rb +32 -41
  74. data/lib/aws-sdk-s3/resource.rb +139 -12
  75. data/lib/aws-sdk-s3/transfer_manager.rb +304 -0
  76. data/lib/aws-sdk-s3/types.rb +10204 -5378
  77. data/lib/aws-sdk-s3.rb +35 -27
  78. data/sig/bucket.rbs +231 -0
  79. data/sig/bucket_acl.rbs +78 -0
  80. data/sig/bucket_cors.rbs +69 -0
  81. data/sig/bucket_lifecycle.rbs +88 -0
  82. data/sig/bucket_lifecycle_configuration.rbs +115 -0
  83. data/sig/bucket_logging.rbs +76 -0
  84. data/sig/bucket_notification.rbs +114 -0
  85. data/sig/bucket_policy.rbs +59 -0
  86. data/sig/bucket_request_payment.rbs +54 -0
  87. data/sig/bucket_tagging.rbs +65 -0
  88. data/sig/bucket_versioning.rbs +77 -0
  89. data/sig/bucket_website.rbs +93 -0
  90. data/sig/client.rbs +2586 -0
  91. data/sig/customizations/bucket.rbs +19 -0
  92. data/sig/customizations/object.rbs +38 -0
  93. data/sig/customizations/object_summary.rbs +35 -0
  94. data/sig/errors.rbs +44 -0
  95. data/sig/multipart_upload.rbs +120 -0
  96. data/sig/multipart_upload_part.rbs +109 -0
  97. data/sig/object.rbs +464 -0
  98. data/sig/object_acl.rbs +86 -0
  99. data/sig/object_summary.rbs +347 -0
  100. data/sig/object_version.rbs +143 -0
  101. data/sig/resource.rbs +141 -0
  102. data/sig/types.rbs +2868 -0
  103. data/sig/waiters.rbs +95 -0
  104. metadata +51 -16
  105. data/lib/aws-sdk-s3/arn/access_point_arn.rb +0 -69
  106. data/lib/aws-sdk-s3/arn/multi_region_access_point_arn.rb +0 -69
  107. data/lib/aws-sdk-s3/arn/object_lambda_arn.rb +0 -69
  108. data/lib/aws-sdk-s3/arn/outpost_access_point_arn.rb +0 -73
  109. data/lib/aws-sdk-s3/plugins/object_lambda_endpoint.rb +0 -25
@@ -5,22 +5,6 @@ require 'uri'
5
5
  module Aws
6
6
  module S3
7
7
  class Bucket
8
- # Save the old initialize method so that we can call 'super'.
9
- old_initialize = instance_method(:initialize)
10
- # Make the method redefinable
11
- alias_method :initialize, :initialize
12
- # Define a new initialize method that extracts out a bucket ARN.
13
- define_method(:initialize) do |*args|
14
- old_initialize.bind(self).call(*args)
15
- resolved_region, arn = Plugins::ARN.resolve_arn!(
16
- name,
17
- client.config.region,
18
- client.config.s3_use_arn_region
19
- )
20
- @resolved_region = resolved_region
21
- @arn = arn
22
- end
23
-
24
8
  # Deletes all objects and versioned objects from this bucket
25
9
  #
26
10
  # @example
@@ -105,14 +89,27 @@ module Aws
105
89
  if options[:virtual_host]
106
90
  scheme = options.fetch(:secure, true) ? 'https' : 'http'
107
91
  "#{scheme}://#{name}"
108
- elsif @arn
109
- Plugins::ARN.resolve_url!(
110
- client.config.endpoint.dup,
111
- @arn,
112
- @resolved_region
113
- ).to_s
114
92
  else
115
- s3_bucket_url
93
+ # Taken from Aws::S3::Endpoints module
94
+ unless client.config.regional_endpoint
95
+ endpoint = client.config.endpoint.to_s
96
+ end
97
+ params = Aws::S3::EndpointParameters.new(
98
+ bucket: name,
99
+ region: client.config.region,
100
+ use_fips: client.config.use_fips_endpoint,
101
+ use_dual_stack: client.config.use_dualstack_endpoint,
102
+ endpoint: endpoint,
103
+ force_path_style: client.config.force_path_style,
104
+ accelerate: client.config.use_accelerate_endpoint,
105
+ use_global_endpoint: client.config.s3_us_east_1_regional_endpoint == 'legacy',
106
+ use_object_lambda_endpoint: nil,
107
+ disable_access_points: nil,
108
+ disable_multi_region_access_points: client.config.s3_disable_multiregion_access_points,
109
+ use_arn_region: client.config.s3_use_arn_region,
110
+ )
111
+ endpoint = Aws::S3::EndpointProvider.new.resolve_endpoint(params)
112
+ endpoint.url
116
113
  end
117
114
  end
118
115
 
@@ -137,34 +134,13 @@ module Aws
137
134
 
138
135
  # @api private
139
136
  def load
140
- @data = client.list_buckets.buckets.find { |b| b.name == name }
137
+ @data = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
138
+ client.list_buckets.buckets.find { |b| b.name == name }
139
+ end
141
140
  raise "unable to load bucket #{name}" if @data.nil?
142
141
 
143
142
  self
144
143
  end
145
-
146
- private
147
-
148
- def s3_bucket_url
149
- url = client.config.endpoint.dup
150
- if bucket_as_hostname?(url.scheme == 'https')
151
- url.host = "#{name}.#{url.host}"
152
- else
153
- url.path += '/' unless url.path[-1] == '/'
154
- url.path += Seahorse::Util.uri_escape(name)
155
- end
156
- if (client.config.region == 'us-east-1') &&
157
- (client.config.s3_us_east_1_regional_endpoint == 'legacy')
158
- url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
159
- end
160
- url.to_s
161
- end
162
-
163
- def bucket_as_hostname?(https)
164
- Plugins::BucketDns.dns_compatible?(name, https) &&
165
- !client.config.force_path_style
166
- end
167
-
168
144
  end
169
145
  end
170
146
  end
@@ -0,0 +1,40 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ module Errors
6
+ # Hijack PermanentRedirect dynamic error to include the bucket, region,
7
+ # and endpoint.
8
+ class PermanentRedirect < ServiceError
9
+ # @param [Seahorse::Client::RequestContext] context
10
+ # @param [String] message
11
+ # @param [Aws::S3::Types::PermanentRedirect] _data
12
+ def initialize(context, message, _data = Aws::EmptyStructure.new)
13
+ data = Aws::S3::Types::PermanentRedirect.new(message: message)
14
+ body = context.http_response.body_contents
15
+ if (endpoint = body.match(/<Endpoint>(.+?)<\/Endpoint>/))
16
+ data.endpoint = endpoint[1]
17
+ end
18
+ if (bucket = body.match(/<Bucket>(.+?)<\/Bucket>/))
19
+ data.bucket = bucket[1]
20
+ end
21
+ data.region = context.http_response.headers['x-amz-bucket-region']
22
+ super(context, message, data)
23
+ end
24
+ end
25
+
26
+ # Hijack PermanentRedirect (HeadBucket case - no body) dynamic error to
27
+ # include the region.
28
+ class Http301Error < ServiceError
29
+ # @param [Seahorse::Client::RequestContext] context
30
+ # @param [String] message
31
+ # @param [Aws::S3::Types::PermanentRedirect] _data
32
+ def initialize(context, message, _data = Aws::EmptyStructure.new)
33
+ data = Aws::S3::Types::PermanentRedirect.new(message: message)
34
+ data.region = context.http_response.headers['x-amz-bucket-region']
35
+ super(context, message, data)
36
+ end
37
+ end
38
+ end
39
+ end
40
+ end
@@ -27,10 +27,13 @@ module Aws
27
27
  # necessary for objects larger than 5GB and can provide
28
28
  # performance improvements on large objects. Amazon S3 does
29
29
  # not accept multipart copies for objects smaller than 5MB.
30
+ # Object metadata such as Content-Type will be copied, however,
31
+ # Checksums are not copied.
30
32
  #
31
33
  # @option options [Integer] :content_length Only used when
32
34
  # `:multipart_copy` is `true`. Passing this options avoids a HEAD
33
- # request to query the source object size. Raises an `ArgumentError` if
35
+ # request to query the source object size but prevents object metadata
36
+ # from being copied. Raises an `ArgumentError` if
34
37
  # this option is provided when `:multipart_copy` is `false` or not set.
35
38
  #
36
39
  # @option options [S3::Client] :copy_source_client Only used when
@@ -43,6 +46,14 @@ module Aws
43
46
  # different region. You do not need to specify this option
44
47
  # if you have provided a `:source_client` or a `:content_length`.
45
48
  #
49
+ # @option options [Boolean] :use_source_parts (false) Only used when
50
+ # `:multipart_copy` is `true`. Use part sizes defined on the source
51
+ # object if any exist. If copying or moving an object that
52
+ # is already multipart, this does not re-part the object, instead
53
+ # re-using the part definitions on the original. That means the etag
54
+ # and any checksums will not change. This is especially useful if the
55
+ # source object has parts with varied sizes.
56
+ #
46
57
  # @example Basic object copy
47
58
  #
48
59
  # bucket = Aws::S3::Bucket.new('target-bucket')
@@ -65,11 +76,13 @@ module Aws
65
76
  # @see #copy_to
66
77
  #
67
78
  def copy_from(source, options = {})
68
- if Hash === source && source[:copy_source]
69
- # for backwards compatibility
70
- @client.copy_object(source.merge(bucket: bucket_name, key: key))
71
- else
72
- ObjectCopier.new(self, options).copy_from(source, options)
79
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
80
+ if Hash === source && source[:copy_source]
81
+ # for backwards compatibility
82
+ @client.copy_object(source.merge(bucket: bucket_name, key: key))
83
+ else
84
+ ObjectCopier.new(self, options).copy_from(source, options)
85
+ end
73
86
  end
74
87
  end
75
88
 
@@ -106,7 +119,9 @@ module Aws
106
119
  # object.copy_to('src-bucket/src-key', multipart_copy: true)
107
120
  #
108
121
  def copy_to(target, options = {})
109
- ObjectCopier.new(self, options).copy_to(target, options)
122
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
123
+ ObjectCopier.new(self, options).copy_to(target, options)
124
+ end
110
125
  end
111
126
 
112
127
  # Copies and deletes the current object. The object will only be deleted
@@ -161,7 +176,7 @@ module Aws
161
176
  #
162
177
  # @param [Symbol] method
163
178
  # The S3 operation to generate a presigned URL for. Valid values
164
- # are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
179
+ # are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
165
180
  # `:list_multipart_uploads`, `:complete_multipart_upload`,
166
181
  # `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
167
182
  #
@@ -215,6 +230,79 @@ module Aws
215
230
  )
216
231
  end
217
232
 
233
+ # Allows you to create presigned URL requests for S3 operations. This
234
+ # method returns a tuple containing the URL and the signed X-amz-* headers
235
+ # to be used with the presigned url.
236
+ #
237
+ # @example Pre-signed GET URL, valid for one hour
238
+ #
239
+ # obj.presigned_request(:get, expires_in: 3600)
240
+ # #=> ["https://bucket-name.s3.amazonaws.com/object-key?...", {}]
241
+ #
242
+ # @example Pre-signed PUT with a canned ACL
243
+ #
244
+ # # the object uploaded using this URL will be publicly accessible
245
+ # obj.presigned_request(:put, acl: 'public-read')
246
+ # #=> ["https://bucket-name.s3.amazonaws.com/object-key?...",
247
+ # {"x-amz-acl"=>"public-read"}]
248
+ #
249
+ # @param [Symbol] method
250
+ # The S3 operation to generate a presigned request for. Valid values
251
+ # are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
252
+ # `:list_multipart_uploads`, `:complete_multipart_upload`,
253
+ # `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
254
+ #
255
+ # @param [Hash] params
256
+ # Additional request parameters to use when generating the pre-signed
257
+ # request. See the related documentation in {Client} for accepted
258
+ # params.
259
+ #
260
+ # | Method | Client Method |
261
+ # |------------------------------|------------------------------------|
262
+ # | `:get` | {Client#get_object} |
263
+ # | `:put` | {Client#put_object} |
264
+ # | `:head` | {Client#head_object} |
265
+ # | `:delete` | {Client#delete_object} |
266
+ # | `:create_multipart_upload` | {Client#create_multipart_upload} |
267
+ # | `:list_multipart_uploads` | {Client#list_multipart_uploads} |
268
+ # | `:complete_multipart_upload` | {Client#complete_multipart_upload} |
269
+ # | `:abort_multipart_upload` | {Client#abort_multipart_upload} |
270
+ # | `:list_parts` | {Client#list_parts} |
271
+ # | `:upload_part` | {Client#upload_part} |
272
+ #
273
+ # @option params [Boolean] :virtual_host (false) When `true` the
274
+ # presigned URL will use the bucket name as a virtual host.
275
+ #
276
+ # bucket = Aws::S3::Bucket.new('my.bucket.com')
277
+ # bucket.object('key').presigned_request(virtual_host: true)
278
+ # #=> ["http://my.bucket.com/key?...", {}]
279
+ #
280
+ # @option params [Integer] :expires_in (900) Number of seconds before
281
+ # the pre-signed URL expires. This may not exceed one week (604800
282
+ # seconds). Note that the pre-signed URL is also only valid as long as
283
+ # credentials used to sign it are. For example, when using IAM roles,
284
+ # temporary tokens generated for signing also have a default expiration
285
+ # which will affect the effective expiration of the pre-signed URL.
286
+ #
287
+ # @raise [ArgumentError] Raised if `:expires_in` exceeds one week
288
+ # (604800 seconds).
289
+ #
290
+ # @return [String, Hash] A tuple with a presigned URL and headers that
291
+ # should be included with the request.
292
+ #
293
+ def presigned_request(method, params = {})
294
+ presigner = Presigner.new(client: client)
295
+
296
+ if %w(delete head get put).include?(method.to_s)
297
+ method = "#{method}_object".to_sym
298
+ end
299
+
300
+ presigner.presigned_request(
301
+ method.downcase,
302
+ params.merge(bucket: bucket_name, key: key)
303
+ )
304
+ end
305
+
218
306
  # Returns the public (un-signed) URL for this object.
219
307
  #
220
308
  # s3.bucket('bucket-name').object('obj-key').public_url
@@ -265,9 +353,13 @@ module Aws
265
353
  # obj.upload_stream do |write_stream|
266
354
  # IO.copy_stream(STDIN, write_stream)
267
355
  # end
356
+ # @param [Hash] options
357
+ # Additional options for {Client#create_multipart_upload},
358
+ # {Client#complete_multipart_upload},
359
+ # and {Client#upload_part} can be provided.
268
360
  #
269
- # @option options [Integer] :thread_count (10) The number of parallel
270
- # multipart uploads
361
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
362
+ # An additional thread is used internally for task coordination.
271
363
  #
272
364
  # @option options [Boolean] :tempfile (false) Normally read data is stored
273
365
  # in memory when building the parts in order to complete the underlying
@@ -287,28 +379,32 @@ module Aws
287
379
  # @return [Boolean] Returns `true` when the object is uploaded
288
380
  # without any errors.
289
381
  #
382
+ # @see Client#create_multipart_upload
383
+ # @see Client#complete_multipart_upload
384
+ # @see Client#upload_part
290
385
  def upload_stream(options = {}, &block)
291
- uploading_options = options.dup
386
+ upload_opts = options.merge(bucket: bucket_name, key: key)
387
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
292
388
  uploader = MultipartStreamUploader.new(
293
389
  client: client,
294
- thread_count: uploading_options.delete(:thread_count),
295
- tempfile: uploading_options.delete(:tempfile),
296
- part_size: uploading_options.delete(:part_size)
297
- )
298
- uploader.upload(
299
- uploading_options.merge(bucket: bucket_name, key: key),
300
- &block
390
+ executor: executor,
391
+ tempfile: upload_opts.delete(:tempfile),
392
+ part_size: upload_opts.delete(:part_size)
301
393
  )
394
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
395
+ uploader.upload(upload_opts, &block)
396
+ end
397
+ executor.shutdown
302
398
  true
303
399
  end
400
+ deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
304
401
 
305
402
  # Uploads a file from disk to the current object in S3.
306
403
  #
307
404
  # # small files are uploaded in a single API call
308
405
  # obj.upload_file('/path/to/file')
309
406
  #
310
- # Files larger than or equal to `:multipart_threshold` are uploaded
311
- # using the Amazon S3 multipart upload APIs.
407
+ # Files larger than or equal to `:multipart_threshold` are uploaded using the Amazon S3 multipart upload APIs.
312
408
  #
313
409
  # # large files are automatically split into parts
314
410
  # # and the parts are uploaded in parallel
@@ -324,92 +420,142 @@ module Aws
324
420
  # You can provide a callback to monitor progress of the upload:
325
421
  #
326
422
  # # bytes and totals are each an array with 1 entry per part
327
- # progress = Proc.new do |bytes, totals|
328
- # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%" }
423
+ # progress = proc do |bytes, totals|
424
+ # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
329
425
  # end
330
- # obj.upload_file('/path/to/file')
426
+ # obj.upload_file('/path/to/file', progress_callback: progress)
331
427
  #
332
- # @param [String, Pathname, File, Tempfile] source A file on the local
333
- # file system that will be uploaded as this object. This can either be
334
- # a String or Pathname to the file, an open File object, or an open
335
- # Tempfile object. If you pass an open File or Tempfile object, then
336
- # you are responsible for closing it after the upload completes. When
337
- # using an open Tempfile, rewind it before uploading or else the object
428
+ # @param [String, Pathname, File, Tempfile] source A file on the local file system that will be uploaded as
429
+ # this object. This can either be a String or Pathname to the file, an open File object, or an open
430
+ # Tempfile object. If you pass an open File or Tempfile object, then you are responsible for closing it
431
+ # after the upload completes. When using an open Tempfile, rewind it before uploading or else the object
338
432
  # will be empty.
339
433
  #
340
- # @option options [Integer] :multipart_threshold (104857600) Files larger
341
- # than or equal to `:multipart_threshold` are uploaded using the S3
342
- # multipart APIs.
343
- # Default threshold is 100MB.
434
+ # @param [Hash] options
435
+ # Additional options for {Client#put_object} when file sizes below the multipart threshold.
436
+ # For files larger than the multipart threshold, options for {Client#create_multipart_upload},
437
+ # {Client#complete_multipart_upload}, and {Client#upload_part} can be provided.
344
438
  #
345
- # @option options [Integer] :thread_count (10) The number of parallel
346
- # multipart uploads. This option is not used if the file is smaller than
347
- # `:multipart_threshold`.
439
+ # @option options [Integer] :multipart_threshold (104857600) Files larger han or equal to
440
+ # `:multipart_threshold` are uploaded using the S3 multipart APIs. Default threshold is 100MB.
441
+ #
442
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
443
+ # This option is not used if the file is smaller than `:multipart_threshold`.
348
444
  #
349
445
  # @option options [Proc] :progress_callback
350
446
  # A Proc that will be called when each chunk of the upload is sent.
351
447
  # It will be invoked with [bytes_read], [total_sizes]
352
448
  #
353
- # @raise [MultipartUploadError] If an object is being uploaded in
354
- # parts, and the upload can not be completed, then the upload is
355
- # aborted and this error is raised. The raised error has a `#errors`
356
- # method that returns the failures that caused the upload to be
357
- # aborted.
449
+ # @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed,
450
+ # then the upload is aborted and this error is raised. The raised error has a `#errors` method that
451
+ # returns the failures that caused the upload to be aborted.
358
452
  #
359
- # @return [Boolean] Returns `true` when the object is uploaded
360
- # without any errors.
453
+ # @return [Boolean] Returns `true` when the object is uploaded without any errors.
454
+ #
455
+ # @see Client#put_object
456
+ # @see Client#create_multipart_upload
457
+ # @see Client#complete_multipart_upload
458
+ # @see Client#upload_part
361
459
  def upload_file(source, options = {})
362
- uploading_options = options.dup
460
+ upload_opts = options.merge(bucket: bucket_name, key: key)
461
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
363
462
  uploader = FileUploader.new(
364
- multipart_threshold: uploading_options.delete(:multipart_threshold),
365
- client: client
366
- )
367
- response = uploader.upload(
368
- source,
369
- uploading_options.merge(bucket: bucket_name, key: key)
463
+ client: client,
464
+ executor: executor,
465
+ multipart_threshold: upload_opts.delete(:multipart_threshold)
370
466
  )
467
+ response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
468
+ uploader.upload(source, upload_opts)
469
+ end
371
470
  yield response if block_given?
471
+ executor.shutdown
372
472
  true
373
473
  end
474
+ deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
374
475
 
375
476
  # Downloads a file in S3 to a path on disk.
376
477
  #
377
478
  # # small files (< 5MB) are downloaded in a single API call
378
479
  # obj.download_file('/path/to/file')
379
480
  #
380
- # Files larger than 5MB are downloaded using multipart method
481
+ # Files larger than 5MB are downloaded using multipart method:
381
482
  #
382
483
  # # large files are split into parts
383
484
  # # and the parts are downloaded in parallel
384
485
  # obj.download_file('/path/to/very_large_file')
385
486
  #
386
- # @param [String] destination Where to download the file to.
487
+ # You can provide a callback to monitor progress of the download:
488
+ #
489
+ # # bytes and part_sizes are each an array with 1 entry per part
490
+ # # part_sizes may not be known until the first bytes are retrieved
491
+ # progress = proc do |bytes, part_sizes, file_size|
492
+ # puts bytes.map.with_index { |b, i| "Part #{i + 1}: #{b} / #{part_sizes[i]}" }.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
493
+ # end
494
+ # obj.download_file('/path/to/file', progress_callback: progress)
495
+ #
496
+ # @param [String, Pathname, File, Tempfile] destination
497
+ # Where to download the file to. This can either be a String or Pathname to the file, an open File object,
498
+ # or an open Tempfile object. If you pass an open File or Tempfile object, then you are responsible for
499
+ # closing it after the download completes. Download behavior varies by destination type:
500
+ #
501
+ # * **String/Pathname paths**: Downloads to a temporary file first, then atomically moves to the final
502
+ # destination. This prevents corruption of any existing file if the download fails.
503
+ # * **File/Tempfile objects**: Downloads directly to the file object without using temporary files.
504
+ # You are responsible for managing the file object's state and closing it after the download completes.
505
+ # If the download fails, the file object may contain partial data.
387
506
  #
388
- # @option options [String] mode `auto`, `single_request`, `get_range`
389
- # `single_request` mode forces only 1 GET request is made in download,
390
- # `get_range` mode allows `chunk_size` parameter to configured in
391
- # customizing each range size in multipart_download,
392
- # By default, `auto` mode is enabled, which performs multipart_download
507
+ # @param [Hash] options
508
+ # Additional options for {Client#get_object} and #{Client#head_object} may be provided.
393
509
  #
394
- # @option options [String] chunk_size required in get_range mode.
510
+ # @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"`
395
511
  #
396
- # @option options [Integer] thread_count (10) Customize threads used in
397
- # the multipart download.
512
+ # * `auto` mode is enabled by default, which performs `multipart_download`
513
+ # * `"single_request`" mode forces only 1 GET request is made in download
514
+ # * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size
398
515
  #
399
- # @option options [String] version_id The object version id used to
400
- # retrieve the object. For more about object versioning, see:
401
- # https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
516
+ # @option options [Integer] :chunk_size required in `"get_range"` mode.
402
517
  #
403
- # @return [Boolean] Returns `true` when the file is downloaded without
404
- # any errors.
518
+ # @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
519
+ #
520
+ # @option options [String] :checksum_mode ("ENABLED")
521
+ # When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will
522
+ # raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated`
523
+ # callback if you need to verify that validation occurred and which algorithm was used.
524
+ # To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
525
+ #
526
+ # @option options [Callable] :on_checksum_validated
527
+ # Called each time a request's checksum is validated with the checksum algorithm and the
528
+ # response. For multipart downloads, this will be called for each part that is downloaded and validated.
529
+ #
530
+ # @option options [Proc] :progress_callback
531
+ # A Proc that will be called when each chunk of the download is received. It will be invoked with
532
+ # `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges),
533
+ # the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes
534
+ # in the part are received.
535
+ #
536
+ # @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors.
537
+ #
538
+ # @return [Boolean] Returns `true` when the file is downloaded without any errors.
539
+ #
540
+ # @see Client#get_object
541
+ # @see Client#head_object
405
542
  def download_file(destination, options = {})
406
- downloader = FileDownloader.new(client: client)
407
- downloader.download(
408
- destination,
409
- options.merge(bucket: bucket_name, key: key)
410
- )
543
+ download_opts = options.merge(bucket: bucket_name, key: key)
544
+ executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
545
+ downloader = FileDownloader.new(client: client, executor: executor)
546
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
547
+ downloader.download(destination, download_opts)
548
+ end
549
+ executor.shutdown
411
550
  true
412
551
  end
552
+ deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
553
+
554
+ class Collection < Aws::Resources::Collection
555
+ alias_method :delete, :batch_delete!
556
+ extend Aws::Deprecations
557
+ deprecated :delete, use: :batch_delete!
558
+ end
413
559
  end
414
560
  end
415
561
  end
@@ -80,6 +80,11 @@ module Aws
80
80
  object.download_file(destination, options)
81
81
  end
82
82
 
83
+ class Collection < Aws::Resources::Collection
84
+ alias_method :delete, :batch_delete!
85
+ extend Aws::Deprecations
86
+ deprecated :delete, use: :batch_delete!
87
+ end
83
88
  end
84
89
  end
85
90
  end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ class ObjectVersion
6
+ class Collection < Aws::Resources::Collection
7
+ alias_method :delete, :batch_delete!
8
+ extend Aws::Deprecations
9
+ deprecated :delete, use: :batch_delete!
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ module Types
6
+ # This error is not modeled.
7
+ #
8
+ # The bucket you are attempting to access must be addressed using the
9
+ # specified endpoint. Please send all future requests to this endpoint.
10
+ #
11
+ # @!attribute [rw] endpoint
12
+ # @return [String]
13
+ #
14
+ # @!attribute [rw] bucket
15
+ # @return [String]
16
+ #
17
+ # @!attribute [rw] message
18
+ # @return [String]
19
+ #
20
+ class PermanentRedirect < Struct.new(:endpoint, :bucket, :region, :message)
21
+ SENSITIVE = []
22
+ include Aws::Structure
23
+ end
24
+ end
25
+ end
26
+ end
@@ -1,34 +1,32 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- # utility classes
4
- require 'aws-sdk-s3/bucket_region_cache'
5
- require 'aws-sdk-s3/encryption'
6
- require 'aws-sdk-s3/encryption_v2'
7
- require 'aws-sdk-s3/file_part'
8
- require 'aws-sdk-s3/file_uploader'
9
- require 'aws-sdk-s3/file_downloader'
10
- require 'aws-sdk-s3/legacy_signer'
11
- require 'aws-sdk-s3/multipart_file_uploader'
12
- require 'aws-sdk-s3/multipart_stream_uploader'
13
- require 'aws-sdk-s3/multipart_upload_error'
14
- require 'aws-sdk-s3/object_copier'
15
- require 'aws-sdk-s3/object_multipart_copier'
16
- require 'aws-sdk-s3/presigned_post'
17
- require 'aws-sdk-s3/presigner'
3
+ module Aws
4
+ module S3
5
+ # utility classes
6
+ autoload :BucketRegionCache, 'aws-sdk-s3/bucket_region_cache'
7
+ autoload :Encryption, 'aws-sdk-s3/encryption'
8
+ autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
9
+ autoload :FilePart, 'aws-sdk-s3/file_part'
10
+ autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
11
+ autoload :FileUploader, 'aws-sdk-s3/file_uploader'
12
+ autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
13
+ autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
14
+ autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
15
+ autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
16
+ autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
17
+ autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
18
+ autoload :ObjectCopier, 'aws-sdk-s3/object_copier'
19
+ autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
20
+ autoload :PresignedPost, 'aws-sdk-s3/presigned_post'
21
+ autoload :Presigner, 'aws-sdk-s3/presigner'
22
+ autoload :TransferManager, 'aws-sdk-s3/transfer_manager'
18
23
 
19
- # customizations to generated classes
20
- require 'aws-sdk-s3/customizations/bucket'
21
- require 'aws-sdk-s3/customizations/object'
22
- require 'aws-sdk-s3/customizations/object_summary'
23
- require 'aws-sdk-s3/customizations/multipart_upload'
24
- require 'aws-sdk-s3/customizations/types/list_object_versions_output'
24
+ # s3 express session auth
25
+ autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials'
26
+ autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider'
25
27
 
26
- [
27
- Aws::S3::Object::Collection,
28
- Aws::S3::ObjectSummary::Collection,
29
- Aws::S3::ObjectVersion::Collection,
30
- ].each do |klass|
31
- klass.send(:alias_method, :delete, :batch_delete!)
32
- klass.extend Aws::Deprecations
33
- klass.send(:deprecated, :delete, use: :batch_delete!)
28
+ # s3 access grants auth
29
+ autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials'
30
+ autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider'
31
+ end
34
32
  end