aws-sdk-s3 1.10.0 → 1.208.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +1517 -0
  3. data/LICENSE.txt +202 -0
  4. data/VERSION +1 -0
  5. data/lib/aws-sdk-s3/access_grants_credentials.rb +57 -0
  6. data/lib/aws-sdk-s3/access_grants_credentials_provider.rb +250 -0
  7. data/lib/aws-sdk-s3/bucket.rb +1062 -99
  8. data/lib/aws-sdk-s3/bucket_acl.rb +67 -17
  9. data/lib/aws-sdk-s3/bucket_cors.rb +80 -17
  10. data/lib/aws-sdk-s3/bucket_lifecycle.rb +71 -19
  11. data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +126 -20
  12. data/lib/aws-sdk-s3/bucket_logging.rb +68 -18
  13. data/lib/aws-sdk-s3/bucket_notification.rb +56 -20
  14. data/lib/aws-sdk-s3/bucket_policy.rb +108 -17
  15. data/lib/aws-sdk-s3/bucket_region_cache.rb +11 -5
  16. data/lib/aws-sdk-s3/bucket_request_payment.rb +60 -15
  17. data/lib/aws-sdk-s3/bucket_tagging.rb +71 -17
  18. data/lib/aws-sdk-s3/bucket_versioning.rb +166 -17
  19. data/lib/aws-sdk-s3/bucket_website.rb +78 -17
  20. data/lib/aws-sdk-s3/client.rb +20068 -3879
  21. data/lib/aws-sdk-s3/client_api.rb +1957 -209
  22. data/lib/aws-sdk-s3/customizations/bucket.rb +57 -38
  23. data/lib/aws-sdk-s3/customizations/errors.rb +40 -0
  24. data/lib/aws-sdk-s3/customizations/multipart_upload.rb +2 -0
  25. data/lib/aws-sdk-s3/customizations/object.rb +338 -68
  26. data/lib/aws-sdk-s3/customizations/object_summary.rb +17 -0
  27. data/lib/aws-sdk-s3/customizations/object_version.rb +13 -0
  28. data/lib/aws-sdk-s3/customizations/types/list_object_versions_output.rb +2 -0
  29. data/lib/aws-sdk-s3/customizations/types/permanent_redirect.rb +26 -0
  30. data/lib/aws-sdk-s3/customizations.rb +30 -27
  31. data/lib/aws-sdk-s3/default_executor.rb +103 -0
  32. data/lib/aws-sdk-s3/encryption/client.rb +29 -8
  33. data/lib/aws-sdk-s3/encryption/decrypt_handler.rb +71 -29
  34. data/lib/aws-sdk-s3/encryption/default_cipher_provider.rb +45 -5
  35. data/lib/aws-sdk-s3/encryption/default_key_provider.rb +2 -0
  36. data/lib/aws-sdk-s3/encryption/encrypt_handler.rb +15 -2
  37. data/lib/aws-sdk-s3/encryption/errors.rb +2 -0
  38. data/lib/aws-sdk-s3/encryption/io_auth_decrypter.rb +11 -3
  39. data/lib/aws-sdk-s3/encryption/io_decrypter.rb +11 -3
  40. data/lib/aws-sdk-s3/encryption/io_encrypter.rb +2 -0
  41. data/lib/aws-sdk-s3/encryption/key_provider.rb +2 -0
  42. data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +48 -11
  43. data/lib/aws-sdk-s3/encryption/materials.rb +8 -6
  44. data/lib/aws-sdk-s3/encryption/utils.rb +25 -0
  45. data/lib/aws-sdk-s3/encryption.rb +4 -0
  46. data/lib/aws-sdk-s3/encryptionV2/client.rb +645 -0
  47. data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +68 -0
  48. data/lib/aws-sdk-s3/encryptionV2/decryption.rb +205 -0
  49. data/lib/aws-sdk-s3/encryptionV2/default_cipher_provider.rb +187 -0
  50. data/lib/aws-sdk-s3/encryptionV2/default_key_provider.rb +40 -0
  51. data/lib/aws-sdk-s3/encryptionV2/encrypt_handler.rb +67 -0
  52. data/lib/aws-sdk-s3/encryptionV2/errors.rb +37 -0
  53. data/lib/aws-sdk-s3/encryptionV2/io_auth_decrypter.rb +58 -0
  54. data/lib/aws-sdk-s3/encryptionV2/io_decrypter.rb +37 -0
  55. data/lib/aws-sdk-s3/encryptionV2/io_encrypter.rb +75 -0
  56. data/lib/aws-sdk-s3/encryptionV2/key_provider.rb +31 -0
  57. data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +181 -0
  58. data/lib/aws-sdk-s3/encryptionV2/materials.rb +60 -0
  59. data/lib/aws-sdk-s3/encryptionV2/utils.rb +108 -0
  60. data/lib/aws-sdk-s3/encryptionV3/client.rb +885 -0
  61. data/lib/aws-sdk-s3/encryptionV3/decrypt_handler.rb +98 -0
  62. data/lib/aws-sdk-s3/encryptionV3/decryption.rb +244 -0
  63. data/lib/aws-sdk-s3/encryptionV3/default_cipher_provider.rb +159 -0
  64. data/lib/aws-sdk-s3/encryptionV3/default_key_provider.rb +35 -0
  65. data/lib/aws-sdk-s3/encryptionV3/encrypt_handler.rb +98 -0
  66. data/lib/aws-sdk-s3/encryptionV3/errors.rb +47 -0
  67. data/lib/aws-sdk-s3/encryptionV3/io_auth_decrypter.rb +60 -0
  68. data/lib/aws-sdk-s3/encryptionV3/io_decrypter.rb +35 -0
  69. data/lib/aws-sdk-s3/encryptionV3/io_encrypter.rb +84 -0
  70. data/lib/aws-sdk-s3/encryptionV3/key_provider.rb +28 -0
  71. data/lib/aws-sdk-s3/encryptionV3/kms_cipher_provider.rb +159 -0
  72. data/lib/aws-sdk-s3/encryptionV3/materials.rb +58 -0
  73. data/lib/aws-sdk-s3/encryptionV3/utils.rb +321 -0
  74. data/lib/aws-sdk-s3/encryption_v2.rb +24 -0
  75. data/lib/aws-sdk-s3/encryption_v3.rb +24 -0
  76. data/lib/aws-sdk-s3/endpoint_parameters.rb +181 -0
  77. data/lib/aws-sdk-s3/endpoint_provider.rb +886 -0
  78. data/lib/aws-sdk-s3/endpoints.rb +1544 -0
  79. data/lib/aws-sdk-s3/errors.rb +181 -1
  80. data/lib/aws-sdk-s3/event_streams.rb +69 -0
  81. data/lib/aws-sdk-s3/express_credentials.rb +55 -0
  82. data/lib/aws-sdk-s3/express_credentials_provider.rb +59 -0
  83. data/lib/aws-sdk-s3/file_downloader.rb +261 -82
  84. data/lib/aws-sdk-s3/file_part.rb +16 -13
  85. data/lib/aws-sdk-s3/file_uploader.rb +37 -22
  86. data/lib/aws-sdk-s3/legacy_signer.rb +19 -26
  87. data/lib/aws-sdk-s3/multipart_download_error.rb +8 -0
  88. data/lib/aws-sdk-s3/multipart_file_uploader.rb +142 -80
  89. data/lib/aws-sdk-s3/multipart_stream_uploader.rb +191 -0
  90. data/lib/aws-sdk-s3/multipart_upload.rb +342 -31
  91. data/lib/aws-sdk-s3/multipart_upload_error.rb +5 -4
  92. data/lib/aws-sdk-s3/multipart_upload_part.rb +387 -47
  93. data/lib/aws-sdk-s3/object.rb +2733 -204
  94. data/lib/aws-sdk-s3/object_acl.rb +112 -25
  95. data/lib/aws-sdk-s3/object_copier.rb +9 -5
  96. data/lib/aws-sdk-s3/object_multipart_copier.rb +50 -23
  97. data/lib/aws-sdk-s3/object_summary.rb +2265 -181
  98. data/lib/aws-sdk-s3/object_version.rb +542 -74
  99. data/lib/aws-sdk-s3/plugins/accelerate.rb +17 -64
  100. data/lib/aws-sdk-s3/plugins/access_grants.rb +178 -0
  101. data/lib/aws-sdk-s3/plugins/arn.rb +70 -0
  102. data/lib/aws-sdk-s3/plugins/bucket_dns.rb +7 -43
  103. data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +20 -3
  104. data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +31 -0
  105. data/lib/aws-sdk-s3/plugins/dualstack.rb +7 -50
  106. data/lib/aws-sdk-s3/plugins/endpoints.rb +86 -0
  107. data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +5 -4
  108. data/lib/aws-sdk-s3/plugins/express_session_auth.rb +88 -0
  109. data/lib/aws-sdk-s3/plugins/get_bucket_location_fix.rb +3 -1
  110. data/lib/aws-sdk-s3/plugins/http_200_errors.rb +62 -17
  111. data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +44 -0
  112. data/lib/aws-sdk-s3/plugins/location_constraint.rb +5 -1
  113. data/lib/aws-sdk-s3/plugins/md5s.rb +14 -67
  114. data/lib/aws-sdk-s3/plugins/redirects.rb +5 -1
  115. data/lib/aws-sdk-s3/plugins/s3_host_id.rb +2 -0
  116. data/lib/aws-sdk-s3/plugins/s3_signer.rb +67 -93
  117. data/lib/aws-sdk-s3/plugins/sse_cpk.rb +3 -1
  118. data/lib/aws-sdk-s3/plugins/streaming_retry.rb +137 -0
  119. data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb +4 -1
  120. data/lib/aws-sdk-s3/presigned_post.rb +160 -99
  121. data/lib/aws-sdk-s3/presigner.rb +178 -81
  122. data/lib/aws-sdk-s3/resource.rb +164 -15
  123. data/lib/aws-sdk-s3/transfer_manager.rb +303 -0
  124. data/lib/aws-sdk-s3/types.rb +15981 -4168
  125. data/lib/aws-sdk-s3/waiters.rb +67 -1
  126. data/lib/aws-sdk-s3.rb +46 -31
  127. data/sig/bucket.rbs +231 -0
  128. data/sig/bucket_acl.rbs +78 -0
  129. data/sig/bucket_cors.rbs +69 -0
  130. data/sig/bucket_lifecycle.rbs +88 -0
  131. data/sig/bucket_lifecycle_configuration.rbs +115 -0
  132. data/sig/bucket_logging.rbs +76 -0
  133. data/sig/bucket_notification.rbs +114 -0
  134. data/sig/bucket_policy.rbs +59 -0
  135. data/sig/bucket_request_payment.rbs +54 -0
  136. data/sig/bucket_tagging.rbs +65 -0
  137. data/sig/bucket_versioning.rbs +77 -0
  138. data/sig/bucket_website.rbs +93 -0
  139. data/sig/client.rbs +2612 -0
  140. data/sig/customizations/bucket.rbs +19 -0
  141. data/sig/customizations/object.rbs +38 -0
  142. data/sig/customizations/object_summary.rbs +35 -0
  143. data/sig/errors.rbs +44 -0
  144. data/sig/multipart_upload.rbs +120 -0
  145. data/sig/multipart_upload_part.rbs +109 -0
  146. data/sig/object.rbs +464 -0
  147. data/sig/object_acl.rbs +86 -0
  148. data/sig/object_summary.rbs +347 -0
  149. data/sig/object_version.rbs +143 -0
  150. data/sig/resource.rbs +141 -0
  151. data/sig/types.rbs +2899 -0
  152. data/sig/waiters.rbs +95 -0
  153. metadata +97 -14
@@ -1,14 +1,19 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  class Object
4
-
5
6
  alias size content_length
6
7
 
8
+ # Make the method redefinable
9
+ alias_method :copy_from, :copy_from
10
+
7
11
  # Copies another object to this object. Use `multipart_copy: true`
8
12
  # for large objects. This is required for objects that exceed 5GB.
9
13
  #
10
- # @param [S3::Object, S3::ObjectVersion, S3::ObjectSummary, String, Hash] source
11
- # Where to copy object data from. `source` must be one of the following:
14
+ # @param [S3::Object, S3::ObjectVersion, S3::ObjectSummary, String, Hash]
15
+ # source Where to copy object data from. `source` must be one of the
16
+ # following:
12
17
  #
13
18
  # * {Aws::S3::Object}
14
19
  # * {Aws::S3::ObjectSummary}
@@ -22,10 +27,14 @@ module Aws
22
27
  # necessary for objects larger than 5GB and can provide
23
28
  # performance improvements on large objects. Amazon S3 does
24
29
  # not accept multipart copies for objects smaller than 5MB.
30
+ # Object metadata such as Content-Type will be copied, however,
31
+ # Checksums are not copied.
25
32
  #
26
33
  # @option options [Integer] :content_length Only used when
27
34
  # `:multipart_copy` is `true`. Passing this options avoids a HEAD
28
- # request to query the source object size.
35
+ # request to query the source object size but prevents object metadata
36
+ # from being copied. Raises an `ArgumentError` if
37
+ # this option is provided when `:multipart_copy` is `false` or not set.
29
38
  #
30
39
  # @option options [S3::Client] :copy_source_client Only used when
31
40
  # `:multipart_copy` is `true` and the source object is in a
@@ -37,6 +46,14 @@ module Aws
37
46
  # different region. You do not need to specify this option
38
47
  # if you have provided a `:source_client` or a `:content_length`.
39
48
  #
49
+ # @option options [Boolean] :use_source_parts (false) Only used when
50
+ # `:multipart_copy` is `true`. Use part sizes defined on the source
51
+ # object if any exist. If copying or moving an object that
52
+ # is already multipart, this does not re-part the object, instead
53
+ # re-using the part definitions on the original. That means the etag
54
+ # and any checksums will not change. This is especially useful if the
55
+ # source object has parts with varied sizes.
56
+ #
40
57
  # @example Basic object copy
41
58
  #
42
59
  # bucket = Aws::S3::Bucket.new('target-bucket')
@@ -59,11 +76,13 @@ module Aws
59
76
  # @see #copy_to
60
77
  #
61
78
  def copy_from(source, options = {})
62
- if Hash === source && source[:copy_source]
63
- # for backwards compatibility
64
- @client.copy_object(source.merge(bucket: bucket_name, key: key))
65
- else
66
- ObjectCopier.new(self, options).copy_from(source, options)
79
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
80
+ if Hash === source && source[:copy_source]
81
+ # for backwards compatibility
82
+ @client.copy_object(source.merge(bucket: bucket_name, key: key))
83
+ else
84
+ ObjectCopier.new(self, options).copy_from(source, options)
85
+ end
67
86
  end
68
87
  end
69
88
 
@@ -100,11 +119,14 @@ module Aws
100
119
  # object.copy_to('src-bucket/src-key', multipart_copy: true)
101
120
  #
102
121
  def copy_to(target, options = {})
103
- ObjectCopier.new(self, options).copy_to(target, options)
122
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
123
+ ObjectCopier.new(self, options).copy_to(target, options)
124
+ end
104
125
  end
105
126
 
106
- # Copies and deletes the current object. The object will only be
107
- # deleted if the copy operation succeeds.
127
+ # Copies and deletes the current object. The object will only be deleted
128
+ # if the copy operation succeeds.
129
+ #
108
130
  # @param (see Object#copy_to)
109
131
  # @option (see Object#copy_to)
110
132
  # @return [void]
@@ -129,10 +151,7 @@ module Aws
129
151
  client.config.credentials,
130
152
  client.config.region,
131
153
  bucket_name,
132
- {
133
- key: key,
134
- url: bucket.url,
135
- }.merge(options)
154
+ { key: key, url: bucket.url }.merge(options)
136
155
  )
137
156
  end
138
157
 
@@ -149,21 +168,35 @@ module Aws
149
168
  # obj.presigned_url(:put, acl: 'public-read')
150
169
  # #=> "https://bucket-name.s3.amazonaws.com/object-key?..."
151
170
  #
152
- # @param [Symbol] http_method
153
- # The HTTP method to generate a presigned URL for. Valid values
154
- # are `:get`, `:put`, `:head`, and `:delete`.
171
+ # @example Pre-signed UploadPart PUT
172
+ #
173
+ # # the object uploaded using this URL will be publicly accessible
174
+ # obj.presigned_url(:upload_part, part_number: 1, upload_id: 'uploadIdToken')
175
+ # #=> "https://bucket-name.s3.amazonaws.com/object-key?..."
176
+ #
177
+ # @param [Symbol] method
178
+ # The S3 operation to generate a presigned URL for. Valid values
179
+ # are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
180
+ # `:list_multipart_uploads`, `:complete_multipart_upload`,
181
+ # `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
155
182
  #
156
183
  # @param [Hash] params
157
184
  # Additional request parameters to use when generating the pre-signed
158
185
  # URL. See the related documentation in {Client} for accepted
159
186
  # params.
160
187
  #
161
- # | HTTP Method | Client Method |
162
- # |---------------|------------------------|
163
- # | `:get` | {Client#get_object} |
164
- # | `:put` | {Client#put_object} |
165
- # | `:head` | {Client#head_object} |
166
- # | `:delete` | {Client#delete_object} |
188
+ # | Method | Client Method |
189
+ # |------------------------------|------------------------------------|
190
+ # | `:get` | {Client#get_object} |
191
+ # | `:put` | {Client#put_object} |
192
+ # | `:head` | {Client#head_object} |
193
+ # | `:delete` | {Client#delete_object} |
194
+ # | `:create_multipart_upload` | {Client#create_multipart_upload} |
195
+ # | `:list_multipart_uploads` | {Client#list_multipart_uploads} |
196
+ # | `:complete_multipart_upload` | {Client#complete_multipart_upload} |
197
+ # | `:abort_multipart_upload` | {Client#abort_multipart_upload} |
198
+ # | `:list_parts` | {Client#list_parts} |
199
+ # | `:upload_part` | {Client#upload_part} |
167
200
  #
168
201
  # @option params [Boolean] :virtual_host (false) When `true` the
169
202
  # presigned URL will use the bucket name as a virtual host.
@@ -184,12 +217,90 @@ module Aws
184
217
  #
185
218
  # @return [String]
186
219
  #
187
- def presigned_url(http_method, params = {})
220
+ def presigned_url(method, params = {})
188
221
  presigner = Presigner.new(client: client)
189
- presigner.presigned_url("#{http_method.downcase}_object", params.merge(
190
- bucket: bucket_name,
191
- key: key,
192
- ))
222
+
223
+ if %w(delete head get put).include?(method.to_s)
224
+ method = "#{method}_object".to_sym
225
+ end
226
+
227
+ presigner.presigned_url(
228
+ method.downcase,
229
+ params.merge(bucket: bucket_name, key: key)
230
+ )
231
+ end
232
+
233
+ # Allows you to create presigned URL requests for S3 operations. This
234
+ # method returns a tuple containing the URL and the signed X-amz-* headers
235
+ # to be used with the presigned url.
236
+ #
237
+ # @example Pre-signed GET URL, valid for one hour
238
+ #
239
+ # obj.presigned_request(:get, expires_in: 3600)
240
+ # #=> ["https://bucket-name.s3.amazonaws.com/object-key?...", {}]
241
+ #
242
+ # @example Pre-signed PUT with a canned ACL
243
+ #
244
+ # # the object uploaded using this URL will be publicly accessible
245
+ # obj.presigned_request(:put, acl: 'public-read')
246
+ # #=> ["https://bucket-name.s3.amazonaws.com/object-key?...",
247
+ # {"x-amz-acl"=>"public-read"}]
248
+ #
249
+ # @param [Symbol] method
250
+ # The S3 operation to generate a presigned request for. Valid values
251
+ # are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
252
+ # `:list_multipart_uploads`, `:complete_multipart_upload`,
253
+ # `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
254
+ #
255
+ # @param [Hash] params
256
+ # Additional request parameters to use when generating the pre-signed
257
+ # request. See the related documentation in {Client} for accepted
258
+ # params.
259
+ #
260
+ # | Method | Client Method |
261
+ # |------------------------------|------------------------------------|
262
+ # | `:get` | {Client#get_object} |
263
+ # | `:put` | {Client#put_object} |
264
+ # | `:head` | {Client#head_object} |
265
+ # | `:delete` | {Client#delete_object} |
266
+ # | `:create_multipart_upload` | {Client#create_multipart_upload} |
267
+ # | `:list_multipart_uploads` | {Client#list_multipart_uploads} |
268
+ # | `:complete_multipart_upload` | {Client#complete_multipart_upload} |
269
+ # | `:abort_multipart_upload` | {Client#abort_multipart_upload} |
270
+ # | `:list_parts` | {Client#list_parts} |
271
+ # | `:upload_part` | {Client#upload_part} |
272
+ #
273
+ # @option params [Boolean] :virtual_host (false) When `true` the
274
+ # presigned URL will use the bucket name as a virtual host.
275
+ #
276
+ # bucket = Aws::S3::Bucket.new('my.bucket.com')
277
+ # bucket.object('key').presigned_request(virtual_host: true)
278
+ # #=> ["http://my.bucket.com/key?...", {}]
279
+ #
280
+ # @option params [Integer] :expires_in (900) Number of seconds before
281
+ # the pre-signed URL expires. This may not exceed one week (604800
282
+ # seconds). Note that the pre-signed URL is also only valid as long as
283
+ # credentials used to sign it are. For example, when using IAM roles,
284
+ # temporary tokens generated for signing also have a default expiration
285
+ # which will affect the effective expiration of the pre-signed URL.
286
+ #
287
+ # @raise [ArgumentError] Raised if `:expires_in` exceeds one week
288
+ # (604800 seconds).
289
+ #
290
+ # @return [String, Hash] A tuple with a presigned URL and headers that
291
+ # should be included with the request.
292
+ #
293
+ def presigned_request(method, params = {})
294
+ presigner = Presigner.new(client: client)
295
+
296
+ if %w(delete head get put).include?(method.to_s)
297
+ method = "#{method}_object".to_sym
298
+ end
299
+
300
+ presigner.presigned_request(
301
+ method.downcase,
302
+ params.merge(bucket: bucket_name, key: key)
303
+ )
193
304
  end
194
305
 
195
306
  # Returns the public (un-signed) URL for this object.
@@ -197,15 +308,22 @@ module Aws
197
308
  # s3.bucket('bucket-name').object('obj-key').public_url
198
309
  # #=> "https://bucket-name.s3.amazonaws.com/obj-key"
199
310
  #
200
- # To use virtual hosted bucket url (disables https):
311
+ # To use virtual hosted bucket url.
312
+ # Uses https unless secure: false is set. If the bucket
313
+ # name contains dots (.) then you will need to set secure: false.
201
314
  #
202
- # s3.bucket('my.bucket.com').object('key').public_url(virtual_host: true)
203
- # #=> "http://my.bucket.com/key"
315
+ # s3.bucket('my-bucket.com').object('key')
316
+ # .public_url(virtual_host: true)
317
+ # #=> "https://my-bucket.com/key"
204
318
  #
205
319
  # @option options [Boolean] :virtual_host (false) When `true`, the bucket
206
320
  # name will be used as the host name. This is useful when you have
207
321
  # a CNAME configured for the bucket.
208
322
  #
323
+ # @option options [Boolean] :secure (true) When `false`, http
324
+ # will be used with virtual_host. This is required when
325
+ # the bucket name has a dot (.) in it.
326
+ #
209
327
  # @return [String]
210
328
  def public_url(options = {})
211
329
  url = URI.parse(bucket.url(options))
@@ -214,77 +332,229 @@ module Aws
214
332
  url.to_s
215
333
  end
216
334
 
335
+ # Uploads a stream in a streaming fashion to the current object in S3.
336
+ #
337
+ # Passed chunks automatically split into multipart upload parts and the
338
+ # parts are uploaded in parallel. This allows for streaming uploads that
339
+ # never touch the disk.
340
+ #
341
+ # Note that this is known to have issues in JRuby until jruby-9.1.15.0,
342
+ # so avoid using this with older versions of JRuby.
343
+ #
344
+ # @example Streaming chunks of data
345
+ # obj.upload_stream do |write_stream|
346
+ # 10.times { write_stream << 'foo' }
347
+ # end
348
+ # @example Streaming chunks of data
349
+ # obj.upload_stream do |write_stream|
350
+ # IO.copy_stream(IO.popen('ls'), write_stream)
351
+ # end
352
+ # @example Streaming chunks of data
353
+ # obj.upload_stream do |write_stream|
354
+ # IO.copy_stream(STDIN, write_stream)
355
+ # end
356
+ # @param [Hash] options
357
+ # Additional options for {Client#create_multipart_upload},
358
+ # {Client#complete_multipart_upload},
359
+ # and {Client#upload_part} can be provided.
360
+ #
361
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
362
+ # An additional thread is used internally for task coordination.
363
+ #
364
+ # @option options [Boolean] :tempfile (false) Normally read data is stored
365
+ # in memory when building the parts in order to complete the underlying
366
+ # multipart upload. By passing `:tempfile => true` data read will be
367
+ # temporarily stored on disk reducing the memory footprint vastly.
368
+ #
369
+ # @option options [Integer] :part_size (5242880)
370
+ # Define how big each part size but the last should be.
371
+ # Default `:part_size` is `5 * 1024 * 1024`.
372
+ #
373
+ # @raise [MultipartUploadError] If an object is being uploaded in
374
+ # parts, and the upload can not be completed, then the upload is
375
+ # aborted and this error is raised. The raised error has a `#errors`
376
+ # method that returns the failures that caused the upload to be
377
+ # aborted.
378
+ #
379
+ # @return [Boolean] Returns `true` when the object is uploaded
380
+ # without any errors.
381
+ #
382
+ # @see Client#create_multipart_upload
383
+ # @see Client#complete_multipart_upload
384
+ # @see Client#upload_part
385
+ def upload_stream(options = {}, &block)
386
+ upload_opts = options.merge(bucket: bucket_name, key: key)
387
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
388
+ uploader = MultipartStreamUploader.new(
389
+ client: client,
390
+ executor: executor,
391
+ tempfile: upload_opts.delete(:tempfile),
392
+ part_size: upload_opts.delete(:part_size)
393
+ )
394
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
395
+ uploader.upload(upload_opts, &block)
396
+ end
397
+ executor.shutdown
398
+ true
399
+ end
400
+ deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
401
+
217
402
  # Uploads a file from disk to the current object in S3.
218
403
  #
219
404
  # # small files are uploaded in a single API call
220
405
  # obj.upload_file('/path/to/file')
221
406
  #
222
- # Files larger than `:multipart_threshold` are uploaded using the
223
- # Amazon S3 multipart upload APIs.
407
+ # Files larger than or equal to `:multipart_threshold` are uploaded using the Amazon S3 multipart upload APIs.
224
408
  #
225
409
  # # large files are automatically split into parts
226
410
  # # and the parts are uploaded in parallel
227
411
  # obj.upload_file('/path/to/very_large_file')
228
412
  #
229
- # @param [String,Pathname,File,Tempfile] source A file or path to a file
230
- # on the local file system that should be uploaded to this object.
231
- # If you pass an open file object, then it is your responsibility
232
- # to close the file object once the upload completes.
413
+ # The response of the S3 upload API is yielded if a block given.
233
414
  #
234
- # @option options [Integer] :multipart_threshold (15728640) Files larger
235
- # than `:multipart_threshold` are uploaded using the S3 multipart APIs.
236
- # Default threshold is 15MB.
415
+ # # API response will have etag value of the file
416
+ # obj.upload_file('/path/to/file') do |response|
417
+ # etag = response.etag
418
+ # end
237
419
  #
238
- # @raise [MultipartUploadError] If an object is being uploaded in
239
- # parts, and the upload can not be completed, then the upload is
240
- # aborted and this error is raised. The raised error has a `#errors`
241
- # method that returns the failures that caused the upload to be
242
- # aborted.
420
+ # You can provide a callback to monitor progress of the upload:
243
421
  #
244
- # @return [Boolean] Returns `true` when the object is uploaded
245
- # without any errors.
422
+ # # bytes and totals are each an array with 1 entry per part
423
+ # progress = proc do |bytes, totals|
424
+ # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
425
+ # end
426
+ # obj.upload_file('/path/to/file', progress_callback: progress)
427
+ #
428
+ # @param [String, Pathname, File, Tempfile] source A file on the local file system that will be uploaded as
429
+ # this object. This can either be a String or Pathname to the file, an open File object, or an open
430
+ # Tempfile object. If you pass an open File or Tempfile object, then you are responsible for closing it
431
+ # after the upload completes. When using an open Tempfile, rewind it before uploading or else the object
432
+ # will be empty.
433
+ #
434
+ # @param [Hash] options
435
+ # Additional options for {Client#put_object} when file sizes below the multipart threshold.
436
+ # For files larger than the multipart threshold, options for {Client#create_multipart_upload},
437
+ # {Client#complete_multipart_upload}, and {Client#upload_part} can be provided.
438
+ #
439
+ # @option options [Integer] :multipart_threshold (104857600) Files larger han or equal to
440
+ # `:multipart_threshold` are uploaded using the S3 multipart APIs. Default threshold is 100MB.
441
+ #
442
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
443
+ # This option is not used if the file is smaller than `:multipart_threshold`.
444
+ #
445
+ # @option options [Proc] :progress_callback
446
+ # A Proc that will be called when each chunk of the upload is sent.
447
+ # It will be invoked with [bytes_read], [total_sizes]
246
448
  #
449
+ # @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed,
450
+ # then the upload is aborted and this error is raised. The raised error has a `#errors` method that
451
+ # returns the failures that caused the upload to be aborted.
452
+ #
453
+ # @return [Boolean] Returns `true` when the object is uploaded without any errors.
454
+ #
455
+ # @see Client#put_object
456
+ # @see Client#create_multipart_upload
457
+ # @see Client#complete_multipart_upload
458
+ # @see Client#upload_part
247
459
  def upload_file(source, options = {})
248
- uploading_options = options.dup
460
+ upload_opts = options.merge(bucket: bucket_name, key: key)
461
+ executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
249
462
  uploader = FileUploader.new(
250
- multipart_threshold: uploading_options.delete(:multipart_threshold),
251
- client: client)
252
- uploader.upload(source, uploading_options.merge(bucket: bucket_name, key: key))
463
+ client: client,
464
+ executor: executor,
465
+ multipart_threshold: upload_opts.delete(:multipart_threshold)
466
+ )
467
+ response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
468
+ uploader.upload(source, upload_opts)
469
+ end
470
+ yield response if block_given?
471
+ executor.shutdown
253
472
  true
254
473
  end
474
+ deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
255
475
 
256
476
  # Downloads a file in S3 to a path on disk.
257
477
  #
258
478
  # # small files (< 5MB) are downloaded in a single API call
259
479
  # obj.download_file('/path/to/file')
260
480
  #
261
- # Files larger than 5MB are downloaded using multipart method
481
+ # Files larger than 5MB are downloaded using multipart method:
262
482
  #
263
483
  # # large files are split into parts
264
484
  # # and the parts are downloaded in parallel
265
485
  # obj.download_file('/path/to/very_large_file')
266
486
  #
267
- # @param [String] destination Where to download the file to
487
+ # You can provide a callback to monitor progress of the download:
268
488
  #
269
- # @option options [String] mode `auto`, `single_request`, `get_range`
270
- # `single_request` mode forces only 1 GET request is made in download,
271
- # `get_range` mode allows `chunk_size` parameter to configured in
272
- # customizing each range size in multipart_download,
273
- # By default, `auto` mode is enabled, which performs multipart_download
489
+ # # bytes and part_sizes are each an array with 1 entry per part
490
+ # # part_sizes may not be known until the first bytes are retrieved
491
+ # progress = proc do |bytes, part_sizes, file_size|
492
+ # puts bytes.map.with_index { |b, i| "Part #{i + 1}: #{b} / #{part_sizes[i]}" }.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
493
+ # end
494
+ # obj.download_file('/path/to/file', progress_callback: progress)
274
495
  #
275
- # @option options [String] chunk_size required in get_range mode
496
+ # @param [String, Pathname, File, Tempfile] destination
497
+ # Where to download the file to. This can either be a String or Pathname to the file, an open File object,
498
+ # or an open Tempfile object. If you pass an open File or Tempfile object, then you are responsible for
499
+ # closing it after the download completes. Download behavior varies by destination type:
276
500
  #
277
- # @option options [String] thread_count Customize threads used in multipart
278
- # download, if not provided, 10 is default value
501
+ # * **String/Pathname paths**: Downloads to a temporary file first, then atomically moves to the final
502
+ # destination. This prevents corruption of any existing file if the download fails.
503
+ # * **File/Tempfile objects**: Downloads directly to the file object without using temporary files.
504
+ # You are responsible for managing the file object's state and closing it after the download completes.
505
+ # If the download fails, the file object may contain partial data.
279
506
  #
280
- # @return [Boolean] Returns `true` when the file is downloaded
281
- # without any errors.
507
+ # @param [Hash] options
508
+ # Additional options for {Client#get_object} and #{Client#head_object} may be provided.
509
+ #
510
+ # @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"`
511
+ #
512
+ # * `auto` mode is enabled by default, which performs `multipart_download`
513
+ # * `"single_request`" mode forces only 1 GET request is made in download
514
+ # * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size
515
+ #
516
+ # @option options [Integer] :chunk_size required in `"get_range"` mode.
517
+ #
518
+ # @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
519
+ #
520
+ # @option options [String] :checksum_mode ("ENABLED")
521
+ # This option is deprecated. Use `:response_checksum_validation` on your S3 client instead.
522
+ # To disable checksum validation, set `response_checksum_validation: 'when_required'`
523
+ # when creating your S3 client.
524
+ #
525
+ # @option options [Callable] :on_checksum_validated
526
+ # Called each time a request's checksum is validated with the checksum algorithm and the
527
+ # response. For multipart downloads, this will be called for each part that is downloaded and validated.
528
+ #
529
+ # @option options [Proc] :progress_callback
530
+ # A Proc that will be called when each chunk of the download is received. It will be invoked with
531
+ # `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges),
532
+ # the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes
533
+ # in the part are received.
534
+ #
535
+ # @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors.
536
+ #
537
+ # @return [Boolean] Returns `true` when the file is downloaded without any errors.
538
+ #
539
+ # @see Client#get_object
540
+ # @see Client#head_object
282
541
  def download_file(destination, options = {})
283
- downloader = FileDownloader.new(client: client)
284
- downloader.download(
285
- destination, options.merge(bucket: bucket_name, key: key))
542
+ download_opts = options.merge(bucket: bucket_name, key: key)
543
+ executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
544
+ downloader = FileDownloader.new(client: client, executor: executor)
545
+ Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
546
+ downloader.download(destination, download_opts)
547
+ end
548
+ executor.shutdown
286
549
  true
287
550
  end
551
+ deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
552
+
553
+ class Collection < Aws::Resources::Collection
554
+ alias_method :delete, :batch_delete!
555
+ extend Aws::Deprecations
556
+ deprecated :delete, use: :batch_delete!
557
+ end
288
558
  end
289
559
  end
290
560
  end
@@ -1,9 +1,14 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Aws
2
4
  module S3
3
5
  class ObjectSummary
4
6
 
5
7
  alias content_length size
6
8
 
9
+ # Make the method redefinable
10
+ alias_method :copy_from, :copy_from
11
+
7
12
  # @param (see Object#copy_from)
8
13
  # @options (see Object#copy_from)
9
14
  # @return (see Object#copy_from)
@@ -60,6 +65,13 @@ module Aws
60
65
  object.upload_file(source, options)
61
66
  end
62
67
 
68
+ # @options (see Object#upload_stream)
69
+ # @return (see Object#upload_stream)
70
+ # @see Object#upload_stream
71
+ def upload_stream(options = {}, &block)
72
+ object.upload_stream(options, &block)
73
+ end
74
+
63
75
  # @param (see Object#download_file)
64
76
  # @options (see Object#download_file)
65
77
  # @return (see Object#download_file)
@@ -68,6 +80,11 @@ module Aws
68
80
  object.download_file(destination, options)
69
81
  end
70
82
 
83
+ class Collection < Aws::Resources::Collection
84
+ alias_method :delete, :batch_delete!
85
+ extend Aws::Deprecations
86
+ deprecated :delete, use: :batch_delete!
87
+ end
71
88
  end
72
89
  end
73
90
  end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ class ObjectVersion
6
+ class Collection < Aws::Resources::Collection
7
+ alias_method :delete, :batch_delete!
8
+ extend Aws::Deprecations
9
+ deprecated :delete, use: :batch_delete!
10
+ end
11
+ end
12
+ end
13
+ end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  class Aws::S3::Types::ListObjectVersionsOutput
2
4
 
3
5
  # TODO : Remove this customization once the resource code
@@ -0,0 +1,26 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Aws
4
+ module S3
5
+ module Types
6
+ # This error is not modeled.
7
+ #
8
+ # The bucket you are attempting to access must be addressed using the
9
+ # specified endpoint. Please send all future requests to this endpoint.
10
+ #
11
+ # @!attribute [rw] endpoint
12
+ # @return [String]
13
+ #
14
+ # @!attribute [rw] bucket
15
+ # @return [String]
16
+ #
17
+ # @!attribute [rw] message
18
+ # @return [String]
19
+ #
20
+ class PermanentRedirect < Struct.new(:endpoint, :bucket, :region, :message)
21
+ SENSITIVE = []
22
+ include Aws::Structure
23
+ end
24
+ end
25
+ end
26
+ end