aws-sdk-s3 1.103.0 → 1.202.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +664 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-s3/access_grants_credentials.rb +57 -0
- data/lib/aws-sdk-s3/access_grants_credentials_provider.rb +250 -0
- data/lib/aws-sdk-s3/bucket.rb +858 -116
- data/lib/aws-sdk-s3/bucket_acl.rb +32 -9
- data/lib/aws-sdk-s3/bucket_cors.rb +38 -13
- data/lib/aws-sdk-s3/bucket_lifecycle.rb +43 -12
- data/lib/aws-sdk-s3/bucket_lifecycle_configuration.rb +100 -13
- data/lib/aws-sdk-s3/bucket_logging.rb +35 -6
- data/lib/aws-sdk-s3/bucket_notification.rb +27 -9
- data/lib/aws-sdk-s3/bucket_policy.rb +79 -10
- data/lib/aws-sdk-s3/bucket_region_cache.rb +9 -5
- data/lib/aws-sdk-s3/bucket_request_payment.rb +29 -7
- data/lib/aws-sdk-s3/bucket_tagging.rb +35 -11
- data/lib/aws-sdk-s3/bucket_versioning.rb +108 -17
- data/lib/aws-sdk-s3/bucket_website.rb +35 -11
- data/lib/aws-sdk-s3/client.rb +11799 -3636
- data/lib/aws-sdk-s3/client_api.rb +1201 -276
- data/lib/aws-sdk-s3/customizations/bucket.rb +23 -47
- data/lib/aws-sdk-s3/customizations/errors.rb +40 -0
- data/lib/aws-sdk-s3/customizations/object.rb +216 -70
- data/lib/aws-sdk-s3/customizations/object_summary.rb +5 -0
- data/lib/aws-sdk-s3/customizations/object_version.rb +13 -0
- data/lib/aws-sdk-s3/customizations/types/permanent_redirect.rb +26 -0
- data/lib/aws-sdk-s3/customizations.rb +27 -29
- data/lib/aws-sdk-s3/default_executor.rb +103 -0
- data/lib/aws-sdk-s3/encryption/client.rb +6 -2
- data/lib/aws-sdk-s3/encryption/kms_cipher_provider.rb +13 -9
- data/lib/aws-sdk-s3/encryptionV2/client.rb +6 -2
- data/lib/aws-sdk-s3/encryptionV2/decrypt_handler.rb +1 -0
- data/lib/aws-sdk-s3/encryptionV2/kms_cipher_provider.rb +10 -6
- data/lib/aws-sdk-s3/endpoint_parameters.rb +181 -0
- data/lib/aws-sdk-s3/endpoint_provider.rb +716 -0
- data/lib/aws-sdk-s3/endpoints.rb +1518 -0
- data/lib/aws-sdk-s3/errors.rb +58 -0
- data/lib/aws-sdk-s3/express_credentials.rb +55 -0
- data/lib/aws-sdk-s3/express_credentials_provider.rb +59 -0
- data/lib/aws-sdk-s3/file_downloader.rb +241 -87
- data/lib/aws-sdk-s3/file_uploader.rb +16 -13
- data/lib/aws-sdk-s3/legacy_signer.rb +2 -1
- data/lib/aws-sdk-s3/multipart_download_error.rb +8 -0
- data/lib/aws-sdk-s3/multipart_file_uploader.rb +108 -86
- data/lib/aws-sdk-s3/multipart_stream_uploader.rb +110 -92
- data/lib/aws-sdk-s3/multipart_upload.rb +294 -19
- data/lib/aws-sdk-s3/multipart_upload_error.rb +3 -4
- data/lib/aws-sdk-s3/multipart_upload_part.rb +297 -31
- data/lib/aws-sdk-s3/object.rb +2224 -269
- data/lib/aws-sdk-s3/object_acl.rb +59 -17
- data/lib/aws-sdk-s3/object_copier.rb +7 -5
- data/lib/aws-sdk-s3/object_multipart_copier.rb +48 -23
- data/lib/aws-sdk-s3/object_summary.rb +1915 -220
- data/lib/aws-sdk-s3/object_version.rb +450 -58
- data/lib/aws-sdk-s3/plugins/accelerate.rb +3 -44
- data/lib/aws-sdk-s3/plugins/access_grants.rb +178 -0
- data/lib/aws-sdk-s3/plugins/arn.rb +0 -197
- data/lib/aws-sdk-s3/plugins/bucket_dns.rb +3 -39
- data/lib/aws-sdk-s3/plugins/bucket_name_restrictions.rb +1 -6
- data/lib/aws-sdk-s3/plugins/checksum_algorithm.rb +31 -0
- data/lib/aws-sdk-s3/plugins/dualstack.rb +1 -55
- data/lib/aws-sdk-s3/plugins/endpoints.rb +86 -0
- data/lib/aws-sdk-s3/plugins/expect_100_continue.rb +2 -1
- data/lib/aws-sdk-s3/plugins/express_session_auth.rb +88 -0
- data/lib/aws-sdk-s3/plugins/http_200_errors.rb +55 -18
- data/lib/aws-sdk-s3/plugins/iad_regional_endpoint.rb +6 -29
- data/lib/aws-sdk-s3/plugins/location_constraint.rb +3 -1
- data/lib/aws-sdk-s3/plugins/md5s.rb +10 -68
- data/lib/aws-sdk-s3/plugins/s3_signer.rb +42 -111
- data/lib/aws-sdk-s3/plugins/streaming_retry.rb +28 -9
- data/lib/aws-sdk-s3/plugins/url_encoded_keys.rb +2 -1
- data/lib/aws-sdk-s3/presigned_post.rb +99 -78
- data/lib/aws-sdk-s3/presigner.rb +32 -41
- data/lib/aws-sdk-s3/resource.rb +139 -12
- data/lib/aws-sdk-s3/transfer_manager.rb +304 -0
- data/lib/aws-sdk-s3/types.rb +10204 -5378
- data/lib/aws-sdk-s3.rb +35 -27
- data/sig/bucket.rbs +231 -0
- data/sig/bucket_acl.rbs +78 -0
- data/sig/bucket_cors.rbs +69 -0
- data/sig/bucket_lifecycle.rbs +88 -0
- data/sig/bucket_lifecycle_configuration.rbs +115 -0
- data/sig/bucket_logging.rbs +76 -0
- data/sig/bucket_notification.rbs +114 -0
- data/sig/bucket_policy.rbs +59 -0
- data/sig/bucket_request_payment.rbs +54 -0
- data/sig/bucket_tagging.rbs +65 -0
- data/sig/bucket_versioning.rbs +77 -0
- data/sig/bucket_website.rbs +93 -0
- data/sig/client.rbs +2586 -0
- data/sig/customizations/bucket.rbs +19 -0
- data/sig/customizations/object.rbs +38 -0
- data/sig/customizations/object_summary.rbs +35 -0
- data/sig/errors.rbs +44 -0
- data/sig/multipart_upload.rbs +120 -0
- data/sig/multipart_upload_part.rbs +109 -0
- data/sig/object.rbs +464 -0
- data/sig/object_acl.rbs +86 -0
- data/sig/object_summary.rbs +347 -0
- data/sig/object_version.rbs +143 -0
- data/sig/resource.rbs +141 -0
- data/sig/types.rbs +2868 -0
- data/sig/waiters.rbs +95 -0
- metadata +51 -16
- data/lib/aws-sdk-s3/arn/access_point_arn.rb +0 -69
- data/lib/aws-sdk-s3/arn/multi_region_access_point_arn.rb +0 -69
- data/lib/aws-sdk-s3/arn/object_lambda_arn.rb +0 -69
- data/lib/aws-sdk-s3/arn/outpost_access_point_arn.rb +0 -73
- data/lib/aws-sdk-s3/plugins/object_lambda_endpoint.rb +0 -25
|
@@ -5,22 +5,6 @@ require 'uri'
|
|
|
5
5
|
module Aws
|
|
6
6
|
module S3
|
|
7
7
|
class Bucket
|
|
8
|
-
# Save the old initialize method so that we can call 'super'.
|
|
9
|
-
old_initialize = instance_method(:initialize)
|
|
10
|
-
# Make the method redefinable
|
|
11
|
-
alias_method :initialize, :initialize
|
|
12
|
-
# Define a new initialize method that extracts out a bucket ARN.
|
|
13
|
-
define_method(:initialize) do |*args|
|
|
14
|
-
old_initialize.bind(self).call(*args)
|
|
15
|
-
resolved_region, arn = Plugins::ARN.resolve_arn!(
|
|
16
|
-
name,
|
|
17
|
-
client.config.region,
|
|
18
|
-
client.config.s3_use_arn_region
|
|
19
|
-
)
|
|
20
|
-
@resolved_region = resolved_region
|
|
21
|
-
@arn = arn
|
|
22
|
-
end
|
|
23
|
-
|
|
24
8
|
# Deletes all objects and versioned objects from this bucket
|
|
25
9
|
#
|
|
26
10
|
# @example
|
|
@@ -105,14 +89,27 @@ module Aws
|
|
|
105
89
|
if options[:virtual_host]
|
|
106
90
|
scheme = options.fetch(:secure, true) ? 'https' : 'http'
|
|
107
91
|
"#{scheme}://#{name}"
|
|
108
|
-
elsif @arn
|
|
109
|
-
Plugins::ARN.resolve_url!(
|
|
110
|
-
client.config.endpoint.dup,
|
|
111
|
-
@arn,
|
|
112
|
-
@resolved_region
|
|
113
|
-
).to_s
|
|
114
92
|
else
|
|
115
|
-
|
|
93
|
+
# Taken from Aws::S3::Endpoints module
|
|
94
|
+
unless client.config.regional_endpoint
|
|
95
|
+
endpoint = client.config.endpoint.to_s
|
|
96
|
+
end
|
|
97
|
+
params = Aws::S3::EndpointParameters.new(
|
|
98
|
+
bucket: name,
|
|
99
|
+
region: client.config.region,
|
|
100
|
+
use_fips: client.config.use_fips_endpoint,
|
|
101
|
+
use_dual_stack: client.config.use_dualstack_endpoint,
|
|
102
|
+
endpoint: endpoint,
|
|
103
|
+
force_path_style: client.config.force_path_style,
|
|
104
|
+
accelerate: client.config.use_accelerate_endpoint,
|
|
105
|
+
use_global_endpoint: client.config.s3_us_east_1_regional_endpoint == 'legacy',
|
|
106
|
+
use_object_lambda_endpoint: nil,
|
|
107
|
+
disable_access_points: nil,
|
|
108
|
+
disable_multi_region_access_points: client.config.s3_disable_multiregion_access_points,
|
|
109
|
+
use_arn_region: client.config.s3_use_arn_region,
|
|
110
|
+
)
|
|
111
|
+
endpoint = Aws::S3::EndpointProvider.new.resolve_endpoint(params)
|
|
112
|
+
endpoint.url
|
|
116
113
|
end
|
|
117
114
|
end
|
|
118
115
|
|
|
@@ -137,34 +134,13 @@ module Aws
|
|
|
137
134
|
|
|
138
135
|
# @api private
|
|
139
136
|
def load
|
|
140
|
-
@data =
|
|
137
|
+
@data = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
138
|
+
client.list_buckets.buckets.find { |b| b.name == name }
|
|
139
|
+
end
|
|
141
140
|
raise "unable to load bucket #{name}" if @data.nil?
|
|
142
141
|
|
|
143
142
|
self
|
|
144
143
|
end
|
|
145
|
-
|
|
146
|
-
private
|
|
147
|
-
|
|
148
|
-
def s3_bucket_url
|
|
149
|
-
url = client.config.endpoint.dup
|
|
150
|
-
if bucket_as_hostname?(url.scheme == 'https')
|
|
151
|
-
url.host = "#{name}.#{url.host}"
|
|
152
|
-
else
|
|
153
|
-
url.path += '/' unless url.path[-1] == '/'
|
|
154
|
-
url.path += Seahorse::Util.uri_escape(name)
|
|
155
|
-
end
|
|
156
|
-
if (client.config.region == 'us-east-1') &&
|
|
157
|
-
(client.config.s3_us_east_1_regional_endpoint == 'legacy')
|
|
158
|
-
url.host = Plugins::IADRegionalEndpoint.legacy_host(url.host)
|
|
159
|
-
end
|
|
160
|
-
url.to_s
|
|
161
|
-
end
|
|
162
|
-
|
|
163
|
-
def bucket_as_hostname?(https)
|
|
164
|
-
Plugins::BucketDns.dns_compatible?(name, https) &&
|
|
165
|
-
!client.config.force_path_style
|
|
166
|
-
end
|
|
167
|
-
|
|
168
144
|
end
|
|
169
145
|
end
|
|
170
146
|
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aws
|
|
4
|
+
module S3
|
|
5
|
+
module Errors
|
|
6
|
+
# Hijack PermanentRedirect dynamic error to include the bucket, region,
|
|
7
|
+
# and endpoint.
|
|
8
|
+
class PermanentRedirect < ServiceError
|
|
9
|
+
# @param [Seahorse::Client::RequestContext] context
|
|
10
|
+
# @param [String] message
|
|
11
|
+
# @param [Aws::S3::Types::PermanentRedirect] _data
|
|
12
|
+
def initialize(context, message, _data = Aws::EmptyStructure.new)
|
|
13
|
+
data = Aws::S3::Types::PermanentRedirect.new(message: message)
|
|
14
|
+
body = context.http_response.body_contents
|
|
15
|
+
if (endpoint = body.match(/<Endpoint>(.+?)<\/Endpoint>/))
|
|
16
|
+
data.endpoint = endpoint[1]
|
|
17
|
+
end
|
|
18
|
+
if (bucket = body.match(/<Bucket>(.+?)<\/Bucket>/))
|
|
19
|
+
data.bucket = bucket[1]
|
|
20
|
+
end
|
|
21
|
+
data.region = context.http_response.headers['x-amz-bucket-region']
|
|
22
|
+
super(context, message, data)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
# Hijack PermanentRedirect (HeadBucket case - no body) dynamic error to
|
|
27
|
+
# include the region.
|
|
28
|
+
class Http301Error < ServiceError
|
|
29
|
+
# @param [Seahorse::Client::RequestContext] context
|
|
30
|
+
# @param [String] message
|
|
31
|
+
# @param [Aws::S3::Types::PermanentRedirect] _data
|
|
32
|
+
def initialize(context, message, _data = Aws::EmptyStructure.new)
|
|
33
|
+
data = Aws::S3::Types::PermanentRedirect.new(message: message)
|
|
34
|
+
data.region = context.http_response.headers['x-amz-bucket-region']
|
|
35
|
+
super(context, message, data)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -27,10 +27,13 @@ module Aws
|
|
|
27
27
|
# necessary for objects larger than 5GB and can provide
|
|
28
28
|
# performance improvements on large objects. Amazon S3 does
|
|
29
29
|
# not accept multipart copies for objects smaller than 5MB.
|
|
30
|
+
# Object metadata such as Content-Type will be copied, however,
|
|
31
|
+
# Checksums are not copied.
|
|
30
32
|
#
|
|
31
33
|
# @option options [Integer] :content_length Only used when
|
|
32
34
|
# `:multipart_copy` is `true`. Passing this options avoids a HEAD
|
|
33
|
-
# request to query the source object size
|
|
35
|
+
# request to query the source object size but prevents object metadata
|
|
36
|
+
# from being copied. Raises an `ArgumentError` if
|
|
34
37
|
# this option is provided when `:multipart_copy` is `false` or not set.
|
|
35
38
|
#
|
|
36
39
|
# @option options [S3::Client] :copy_source_client Only used when
|
|
@@ -43,6 +46,14 @@ module Aws
|
|
|
43
46
|
# different region. You do not need to specify this option
|
|
44
47
|
# if you have provided a `:source_client` or a `:content_length`.
|
|
45
48
|
#
|
|
49
|
+
# @option options [Boolean] :use_source_parts (false) Only used when
|
|
50
|
+
# `:multipart_copy` is `true`. Use part sizes defined on the source
|
|
51
|
+
# object if any exist. If copying or moving an object that
|
|
52
|
+
# is already multipart, this does not re-part the object, instead
|
|
53
|
+
# re-using the part definitions on the original. That means the etag
|
|
54
|
+
# and any checksums will not change. This is especially useful if the
|
|
55
|
+
# source object has parts with varied sizes.
|
|
56
|
+
#
|
|
46
57
|
# @example Basic object copy
|
|
47
58
|
#
|
|
48
59
|
# bucket = Aws::S3::Bucket.new('target-bucket')
|
|
@@ -65,11 +76,13 @@ module Aws
|
|
|
65
76
|
# @see #copy_to
|
|
66
77
|
#
|
|
67
78
|
def copy_from(source, options = {})
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
79
|
+
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
80
|
+
if Hash === source && source[:copy_source]
|
|
81
|
+
# for backwards compatibility
|
|
82
|
+
@client.copy_object(source.merge(bucket: bucket_name, key: key))
|
|
83
|
+
else
|
|
84
|
+
ObjectCopier.new(self, options).copy_from(source, options)
|
|
85
|
+
end
|
|
73
86
|
end
|
|
74
87
|
end
|
|
75
88
|
|
|
@@ -106,7 +119,9 @@ module Aws
|
|
|
106
119
|
# object.copy_to('src-bucket/src-key', multipart_copy: true)
|
|
107
120
|
#
|
|
108
121
|
def copy_to(target, options = {})
|
|
109
|
-
|
|
122
|
+
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
123
|
+
ObjectCopier.new(self, options).copy_to(target, options)
|
|
124
|
+
end
|
|
110
125
|
end
|
|
111
126
|
|
|
112
127
|
# Copies and deletes the current object. The object will only be deleted
|
|
@@ -161,7 +176,7 @@ module Aws
|
|
|
161
176
|
#
|
|
162
177
|
# @param [Symbol] method
|
|
163
178
|
# The S3 operation to generate a presigned URL for. Valid values
|
|
164
|
-
# are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
|
|
179
|
+
# are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
|
|
165
180
|
# `:list_multipart_uploads`, `:complete_multipart_upload`,
|
|
166
181
|
# `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
|
|
167
182
|
#
|
|
@@ -215,6 +230,79 @@ module Aws
|
|
|
215
230
|
)
|
|
216
231
|
end
|
|
217
232
|
|
|
233
|
+
# Allows you to create presigned URL requests for S3 operations. This
|
|
234
|
+
# method returns a tuple containing the URL and the signed X-amz-* headers
|
|
235
|
+
# to be used with the presigned url.
|
|
236
|
+
#
|
|
237
|
+
# @example Pre-signed GET URL, valid for one hour
|
|
238
|
+
#
|
|
239
|
+
# obj.presigned_request(:get, expires_in: 3600)
|
|
240
|
+
# #=> ["https://bucket-name.s3.amazonaws.com/object-key?...", {}]
|
|
241
|
+
#
|
|
242
|
+
# @example Pre-signed PUT with a canned ACL
|
|
243
|
+
#
|
|
244
|
+
# # the object uploaded using this URL will be publicly accessible
|
|
245
|
+
# obj.presigned_request(:put, acl: 'public-read')
|
|
246
|
+
# #=> ["https://bucket-name.s3.amazonaws.com/object-key?...",
|
|
247
|
+
# {"x-amz-acl"=>"public-read"}]
|
|
248
|
+
#
|
|
249
|
+
# @param [Symbol] method
|
|
250
|
+
# The S3 operation to generate a presigned request for. Valid values
|
|
251
|
+
# are `:get`, `:put`, `:head`, `:delete`, `:create_multipart_upload`,
|
|
252
|
+
# `:list_multipart_uploads`, `:complete_multipart_upload`,
|
|
253
|
+
# `:abort_multipart_upload`, `:list_parts`, and `:upload_part`.
|
|
254
|
+
#
|
|
255
|
+
# @param [Hash] params
|
|
256
|
+
# Additional request parameters to use when generating the pre-signed
|
|
257
|
+
# request. See the related documentation in {Client} for accepted
|
|
258
|
+
# params.
|
|
259
|
+
#
|
|
260
|
+
# | Method | Client Method |
|
|
261
|
+
# |------------------------------|------------------------------------|
|
|
262
|
+
# | `:get` | {Client#get_object} |
|
|
263
|
+
# | `:put` | {Client#put_object} |
|
|
264
|
+
# | `:head` | {Client#head_object} |
|
|
265
|
+
# | `:delete` | {Client#delete_object} |
|
|
266
|
+
# | `:create_multipart_upload` | {Client#create_multipart_upload} |
|
|
267
|
+
# | `:list_multipart_uploads` | {Client#list_multipart_uploads} |
|
|
268
|
+
# | `:complete_multipart_upload` | {Client#complete_multipart_upload} |
|
|
269
|
+
# | `:abort_multipart_upload` | {Client#abort_multipart_upload} |
|
|
270
|
+
# | `:list_parts` | {Client#list_parts} |
|
|
271
|
+
# | `:upload_part` | {Client#upload_part} |
|
|
272
|
+
#
|
|
273
|
+
# @option params [Boolean] :virtual_host (false) When `true` the
|
|
274
|
+
# presigned URL will use the bucket name as a virtual host.
|
|
275
|
+
#
|
|
276
|
+
# bucket = Aws::S3::Bucket.new('my.bucket.com')
|
|
277
|
+
# bucket.object('key').presigned_request(virtual_host: true)
|
|
278
|
+
# #=> ["http://my.bucket.com/key?...", {}]
|
|
279
|
+
#
|
|
280
|
+
# @option params [Integer] :expires_in (900) Number of seconds before
|
|
281
|
+
# the pre-signed URL expires. This may not exceed one week (604800
|
|
282
|
+
# seconds). Note that the pre-signed URL is also only valid as long as
|
|
283
|
+
# credentials used to sign it are. For example, when using IAM roles,
|
|
284
|
+
# temporary tokens generated for signing also have a default expiration
|
|
285
|
+
# which will affect the effective expiration of the pre-signed URL.
|
|
286
|
+
#
|
|
287
|
+
# @raise [ArgumentError] Raised if `:expires_in` exceeds one week
|
|
288
|
+
# (604800 seconds).
|
|
289
|
+
#
|
|
290
|
+
# @return [String, Hash] A tuple with a presigned URL and headers that
|
|
291
|
+
# should be included with the request.
|
|
292
|
+
#
|
|
293
|
+
def presigned_request(method, params = {})
|
|
294
|
+
presigner = Presigner.new(client: client)
|
|
295
|
+
|
|
296
|
+
if %w(delete head get put).include?(method.to_s)
|
|
297
|
+
method = "#{method}_object".to_sym
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
presigner.presigned_request(
|
|
301
|
+
method.downcase,
|
|
302
|
+
params.merge(bucket: bucket_name, key: key)
|
|
303
|
+
)
|
|
304
|
+
end
|
|
305
|
+
|
|
218
306
|
# Returns the public (un-signed) URL for this object.
|
|
219
307
|
#
|
|
220
308
|
# s3.bucket('bucket-name').object('obj-key').public_url
|
|
@@ -265,9 +353,13 @@ module Aws
|
|
|
265
353
|
# obj.upload_stream do |write_stream|
|
|
266
354
|
# IO.copy_stream(STDIN, write_stream)
|
|
267
355
|
# end
|
|
356
|
+
# @param [Hash] options
|
|
357
|
+
# Additional options for {Client#create_multipart_upload},
|
|
358
|
+
# {Client#complete_multipart_upload},
|
|
359
|
+
# and {Client#upload_part} can be provided.
|
|
268
360
|
#
|
|
269
|
-
# @option options [Integer] :thread_count (10) The number of parallel
|
|
270
|
-
#
|
|
361
|
+
# @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
|
|
362
|
+
# An additional thread is used internally for task coordination.
|
|
271
363
|
#
|
|
272
364
|
# @option options [Boolean] :tempfile (false) Normally read data is stored
|
|
273
365
|
# in memory when building the parts in order to complete the underlying
|
|
@@ -287,28 +379,32 @@ module Aws
|
|
|
287
379
|
# @return [Boolean] Returns `true` when the object is uploaded
|
|
288
380
|
# without any errors.
|
|
289
381
|
#
|
|
382
|
+
# @see Client#create_multipart_upload
|
|
383
|
+
# @see Client#complete_multipart_upload
|
|
384
|
+
# @see Client#upload_part
|
|
290
385
|
def upload_stream(options = {}, &block)
|
|
291
|
-
|
|
386
|
+
upload_opts = options.merge(bucket: bucket_name, key: key)
|
|
387
|
+
executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
|
|
292
388
|
uploader = MultipartStreamUploader.new(
|
|
293
389
|
client: client,
|
|
294
|
-
|
|
295
|
-
tempfile:
|
|
296
|
-
part_size:
|
|
297
|
-
)
|
|
298
|
-
uploader.upload(
|
|
299
|
-
uploading_options.merge(bucket: bucket_name, key: key),
|
|
300
|
-
&block
|
|
390
|
+
executor: executor,
|
|
391
|
+
tempfile: upload_opts.delete(:tempfile),
|
|
392
|
+
part_size: upload_opts.delete(:part_size)
|
|
301
393
|
)
|
|
394
|
+
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
395
|
+
uploader.upload(upload_opts, &block)
|
|
396
|
+
end
|
|
397
|
+
executor.shutdown
|
|
302
398
|
true
|
|
303
399
|
end
|
|
400
|
+
deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
|
|
304
401
|
|
|
305
402
|
# Uploads a file from disk to the current object in S3.
|
|
306
403
|
#
|
|
307
404
|
# # small files are uploaded in a single API call
|
|
308
405
|
# obj.upload_file('/path/to/file')
|
|
309
406
|
#
|
|
310
|
-
# Files larger than or equal to `:multipart_threshold` are uploaded
|
|
311
|
-
# using the Amazon S3 multipart upload APIs.
|
|
407
|
+
# Files larger than or equal to `:multipart_threshold` are uploaded using the Amazon S3 multipart upload APIs.
|
|
312
408
|
#
|
|
313
409
|
# # large files are automatically split into parts
|
|
314
410
|
# # and the parts are uploaded in parallel
|
|
@@ -324,92 +420,142 @@ module Aws
|
|
|
324
420
|
# You can provide a callback to monitor progress of the upload:
|
|
325
421
|
#
|
|
326
422
|
# # bytes and totals are each an array with 1 entry per part
|
|
327
|
-
# progress =
|
|
328
|
-
# puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
|
|
423
|
+
# progress = proc do |bytes, totals|
|
|
424
|
+
# puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
|
|
329
425
|
# end
|
|
330
|
-
# obj.upload_file('/path/to/file')
|
|
426
|
+
# obj.upload_file('/path/to/file', progress_callback: progress)
|
|
331
427
|
#
|
|
332
|
-
# @param [String, Pathname, File, Tempfile] source A file on the local
|
|
333
|
-
#
|
|
334
|
-
#
|
|
335
|
-
#
|
|
336
|
-
# you are responsible for closing it after the upload completes. When
|
|
337
|
-
# using an open Tempfile, rewind it before uploading or else the object
|
|
428
|
+
# @param [String, Pathname, File, Tempfile] source A file on the local file system that will be uploaded as
|
|
429
|
+
# this object. This can either be a String or Pathname to the file, an open File object, or an open
|
|
430
|
+
# Tempfile object. If you pass an open File or Tempfile object, then you are responsible for closing it
|
|
431
|
+
# after the upload completes. When using an open Tempfile, rewind it before uploading or else the object
|
|
338
432
|
# will be empty.
|
|
339
433
|
#
|
|
340
|
-
# @
|
|
341
|
-
#
|
|
342
|
-
# multipart
|
|
343
|
-
#
|
|
434
|
+
# @param [Hash] options
|
|
435
|
+
# Additional options for {Client#put_object} when file sizes below the multipart threshold.
|
|
436
|
+
# For files larger than the multipart threshold, options for {Client#create_multipart_upload},
|
|
437
|
+
# {Client#complete_multipart_upload}, and {Client#upload_part} can be provided.
|
|
344
438
|
#
|
|
345
|
-
# @option options [Integer] :
|
|
346
|
-
#
|
|
347
|
-
#
|
|
439
|
+
# @option options [Integer] :multipart_threshold (104857600) Files larger han or equal to
|
|
440
|
+
# `:multipart_threshold` are uploaded using the S3 multipart APIs. Default threshold is 100MB.
|
|
441
|
+
#
|
|
442
|
+
# @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
|
|
443
|
+
# This option is not used if the file is smaller than `:multipart_threshold`.
|
|
348
444
|
#
|
|
349
445
|
# @option options [Proc] :progress_callback
|
|
350
446
|
# A Proc that will be called when each chunk of the upload is sent.
|
|
351
447
|
# It will be invoked with [bytes_read], [total_sizes]
|
|
352
448
|
#
|
|
353
|
-
# @raise [MultipartUploadError] If an object is being uploaded in
|
|
354
|
-
#
|
|
355
|
-
#
|
|
356
|
-
# method that returns the failures that caused the upload to be
|
|
357
|
-
# aborted.
|
|
449
|
+
# @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed,
|
|
450
|
+
# then the upload is aborted and this error is raised. The raised error has a `#errors` method that
|
|
451
|
+
# returns the failures that caused the upload to be aborted.
|
|
358
452
|
#
|
|
359
|
-
# @return [Boolean] Returns `true` when the object is uploaded
|
|
360
|
-
#
|
|
453
|
+
# @return [Boolean] Returns `true` when the object is uploaded without any errors.
|
|
454
|
+
#
|
|
455
|
+
# @see Client#put_object
|
|
456
|
+
# @see Client#create_multipart_upload
|
|
457
|
+
# @see Client#complete_multipart_upload
|
|
458
|
+
# @see Client#upload_part
|
|
361
459
|
def upload_file(source, options = {})
|
|
362
|
-
|
|
460
|
+
upload_opts = options.merge(bucket: bucket_name, key: key)
|
|
461
|
+
executor = DefaultExecutor.new(max_threads: upload_opts.delete(:thread_count))
|
|
363
462
|
uploader = FileUploader.new(
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
response = uploader.upload(
|
|
368
|
-
source,
|
|
369
|
-
uploading_options.merge(bucket: bucket_name, key: key)
|
|
463
|
+
client: client,
|
|
464
|
+
executor: executor,
|
|
465
|
+
multipart_threshold: upload_opts.delete(:multipart_threshold)
|
|
370
466
|
)
|
|
467
|
+
response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
468
|
+
uploader.upload(source, upload_opts)
|
|
469
|
+
end
|
|
371
470
|
yield response if block_given?
|
|
471
|
+
executor.shutdown
|
|
372
472
|
true
|
|
373
473
|
end
|
|
474
|
+
deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
|
|
374
475
|
|
|
375
476
|
# Downloads a file in S3 to a path on disk.
|
|
376
477
|
#
|
|
377
478
|
# # small files (< 5MB) are downloaded in a single API call
|
|
378
479
|
# obj.download_file('/path/to/file')
|
|
379
480
|
#
|
|
380
|
-
# Files larger than 5MB are downloaded using multipart method
|
|
481
|
+
# Files larger than 5MB are downloaded using multipart method:
|
|
381
482
|
#
|
|
382
483
|
# # large files are split into parts
|
|
383
484
|
# # and the parts are downloaded in parallel
|
|
384
485
|
# obj.download_file('/path/to/very_large_file')
|
|
385
486
|
#
|
|
386
|
-
#
|
|
487
|
+
# You can provide a callback to monitor progress of the download:
|
|
488
|
+
#
|
|
489
|
+
# # bytes and part_sizes are each an array with 1 entry per part
|
|
490
|
+
# # part_sizes may not be known until the first bytes are retrieved
|
|
491
|
+
# progress = proc do |bytes, part_sizes, file_size|
|
|
492
|
+
# puts bytes.map.with_index { |b, i| "Part #{i + 1}: #{b} / #{part_sizes[i]}" }.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
|
|
493
|
+
# end
|
|
494
|
+
# obj.download_file('/path/to/file', progress_callback: progress)
|
|
495
|
+
#
|
|
496
|
+
# @param [String, Pathname, File, Tempfile] destination
|
|
497
|
+
# Where to download the file to. This can either be a String or Pathname to the file, an open File object,
|
|
498
|
+
# or an open Tempfile object. If you pass an open File or Tempfile object, then you are responsible for
|
|
499
|
+
# closing it after the download completes. Download behavior varies by destination type:
|
|
500
|
+
#
|
|
501
|
+
# * **String/Pathname paths**: Downloads to a temporary file first, then atomically moves to the final
|
|
502
|
+
# destination. This prevents corruption of any existing file if the download fails.
|
|
503
|
+
# * **File/Tempfile objects**: Downloads directly to the file object without using temporary files.
|
|
504
|
+
# You are responsible for managing the file object's state and closing it after the download completes.
|
|
505
|
+
# If the download fails, the file object may contain partial data.
|
|
387
506
|
#
|
|
388
|
-
# @
|
|
389
|
-
#
|
|
390
|
-
# `get_range` mode allows `chunk_size` parameter to configured in
|
|
391
|
-
# customizing each range size in multipart_download,
|
|
392
|
-
# By default, `auto` mode is enabled, which performs multipart_download
|
|
507
|
+
# @param [Hash] options
|
|
508
|
+
# Additional options for {Client#get_object} and #{Client#head_object} may be provided.
|
|
393
509
|
#
|
|
394
|
-
# @option options [String]
|
|
510
|
+
# @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"`
|
|
395
511
|
#
|
|
396
|
-
#
|
|
397
|
-
#
|
|
512
|
+
# * `auto` mode is enabled by default, which performs `multipart_download`
|
|
513
|
+
# * `"single_request`" mode forces only 1 GET request is made in download
|
|
514
|
+
# * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size
|
|
398
515
|
#
|
|
399
|
-
# @option options [
|
|
400
|
-
# retrieve the object. For more about object versioning, see:
|
|
401
|
-
# https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
|
516
|
+
# @option options [Integer] :chunk_size required in `"get_range"` mode.
|
|
402
517
|
#
|
|
403
|
-
# @
|
|
404
|
-
#
|
|
518
|
+
# @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
|
|
519
|
+
#
|
|
520
|
+
# @option options [String] :checksum_mode ("ENABLED")
|
|
521
|
+
# When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will
|
|
522
|
+
# raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated`
|
|
523
|
+
# callback if you need to verify that validation occurred and which algorithm was used.
|
|
524
|
+
# To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
|
|
525
|
+
#
|
|
526
|
+
# @option options [Callable] :on_checksum_validated
|
|
527
|
+
# Called each time a request's checksum is validated with the checksum algorithm and the
|
|
528
|
+
# response. For multipart downloads, this will be called for each part that is downloaded and validated.
|
|
529
|
+
#
|
|
530
|
+
# @option options [Proc] :progress_callback
|
|
531
|
+
# A Proc that will be called when each chunk of the download is received. It will be invoked with
|
|
532
|
+
# `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges),
|
|
533
|
+
# the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes
|
|
534
|
+
# in the part are received.
|
|
535
|
+
#
|
|
536
|
+
# @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors.
|
|
537
|
+
#
|
|
538
|
+
# @return [Boolean] Returns `true` when the file is downloaded without any errors.
|
|
539
|
+
#
|
|
540
|
+
# @see Client#get_object
|
|
541
|
+
# @see Client#head_object
|
|
405
542
|
def download_file(destination, options = {})
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
543
|
+
download_opts = options.merge(bucket: bucket_name, key: key)
|
|
544
|
+
executor = DefaultExecutor.new(max_threads: download_opts.delete([:thread_count]))
|
|
545
|
+
downloader = FileDownloader.new(client: client, executor: executor)
|
|
546
|
+
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
|
547
|
+
downloader.download(destination, download_opts)
|
|
548
|
+
end
|
|
549
|
+
executor.shutdown
|
|
411
550
|
true
|
|
412
551
|
end
|
|
552
|
+
deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
|
|
553
|
+
|
|
554
|
+
class Collection < Aws::Resources::Collection
|
|
555
|
+
alias_method :delete, :batch_delete!
|
|
556
|
+
extend Aws::Deprecations
|
|
557
|
+
deprecated :delete, use: :batch_delete!
|
|
558
|
+
end
|
|
413
559
|
end
|
|
414
560
|
end
|
|
415
561
|
end
|
|
@@ -80,6 +80,11 @@ module Aws
|
|
|
80
80
|
object.download_file(destination, options)
|
|
81
81
|
end
|
|
82
82
|
|
|
83
|
+
class Collection < Aws::Resources::Collection
|
|
84
|
+
alias_method :delete, :batch_delete!
|
|
85
|
+
extend Aws::Deprecations
|
|
86
|
+
deprecated :delete, use: :batch_delete!
|
|
87
|
+
end
|
|
83
88
|
end
|
|
84
89
|
end
|
|
85
90
|
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aws
|
|
4
|
+
module S3
|
|
5
|
+
class ObjectVersion
|
|
6
|
+
class Collection < Aws::Resources::Collection
|
|
7
|
+
alias_method :delete, :batch_delete!
|
|
8
|
+
extend Aws::Deprecations
|
|
9
|
+
deprecated :delete, use: :batch_delete!
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Aws
|
|
4
|
+
module S3
|
|
5
|
+
module Types
|
|
6
|
+
# This error is not modeled.
|
|
7
|
+
#
|
|
8
|
+
# The bucket you are attempting to access must be addressed using the
|
|
9
|
+
# specified endpoint. Please send all future requests to this endpoint.
|
|
10
|
+
#
|
|
11
|
+
# @!attribute [rw] endpoint
|
|
12
|
+
# @return [String]
|
|
13
|
+
#
|
|
14
|
+
# @!attribute [rw] bucket
|
|
15
|
+
# @return [String]
|
|
16
|
+
#
|
|
17
|
+
# @!attribute [rw] message
|
|
18
|
+
# @return [String]
|
|
19
|
+
#
|
|
20
|
+
class PermanentRedirect < Struct.new(:endpoint, :bucket, :region, :message)
|
|
21
|
+
SENSITIVE = []
|
|
22
|
+
include Aws::Structure
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -1,34 +1,32 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
3
|
+
module Aws
|
|
4
|
+
module S3
|
|
5
|
+
# utility classes
|
|
6
|
+
autoload :BucketRegionCache, 'aws-sdk-s3/bucket_region_cache'
|
|
7
|
+
autoload :Encryption, 'aws-sdk-s3/encryption'
|
|
8
|
+
autoload :EncryptionV2, 'aws-sdk-s3/encryption_v2'
|
|
9
|
+
autoload :FilePart, 'aws-sdk-s3/file_part'
|
|
10
|
+
autoload :DefaultExecutor, 'aws-sdk-s3/default_executor'
|
|
11
|
+
autoload :FileUploader, 'aws-sdk-s3/file_uploader'
|
|
12
|
+
autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
|
|
13
|
+
autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
|
|
14
|
+
autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
|
|
15
|
+
autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
|
|
16
|
+
autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
|
|
17
|
+
autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
|
|
18
|
+
autoload :ObjectCopier, 'aws-sdk-s3/object_copier'
|
|
19
|
+
autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
|
|
20
|
+
autoload :PresignedPost, 'aws-sdk-s3/presigned_post'
|
|
21
|
+
autoload :Presigner, 'aws-sdk-s3/presigner'
|
|
22
|
+
autoload :TransferManager, 'aws-sdk-s3/transfer_manager'
|
|
18
23
|
|
|
19
|
-
#
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
require 'aws-sdk-s3/customizations/object_summary'
|
|
23
|
-
require 'aws-sdk-s3/customizations/multipart_upload'
|
|
24
|
-
require 'aws-sdk-s3/customizations/types/list_object_versions_output'
|
|
24
|
+
# s3 express session auth
|
|
25
|
+
autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials'
|
|
26
|
+
autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider'
|
|
25
27
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
].each do |klass|
|
|
31
|
-
klass.send(:alias_method, :delete, :batch_delete!)
|
|
32
|
-
klass.extend Aws::Deprecations
|
|
33
|
-
klass.send(:deprecated, :delete, use: :batch_delete!)
|
|
28
|
+
# s3 access grants auth
|
|
29
|
+
autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials'
|
|
30
|
+
autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider'
|
|
31
|
+
end
|
|
34
32
|
end
|