aws-sdk-s3 1.193.0 → 1.198.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +40 -0
- data/VERSION +1 -1
- data/lib/aws-sdk-s3/client.rb +24 -17
- data/lib/aws-sdk-s3/customizations/object.rb +57 -76
- data/lib/aws-sdk-s3/customizations.rb +2 -1
- data/lib/aws-sdk-s3/file_downloader.rb +57 -74
- data/lib/aws-sdk-s3/file_uploader.rb +3 -5
- data/lib/aws-sdk-s3/multipart_download_error.rb +8 -0
- data/lib/aws-sdk-s3/multipart_file_uploader.rb +34 -65
- data/lib/aws-sdk-s3/multipart_stream_uploader.rb +80 -88
- data/lib/aws-sdk-s3/multipart_upload_error.rb +3 -4
- data/lib/aws-sdk-s3/transfer_manager.rb +246 -0
- data/lib/aws-sdk-s3.rb +1 -1
- data/sig/client.rbs +1 -0
- data/sig/resource.rbs +1 -0
- metadata +5 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8f8abf2a326b6ebef3c9c9ceac08c2378c4e0d9bc3e187a4a3dd89bb81fb8154
|
4
|
+
data.tar.gz: 6186fe037f7fc5541c2a8e1006c9cb0a34630c69fb4a0bdebc40a63b872c9189
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5aade0fd0688cee874c30704211d6f166ea8aaca4838adcdd2ad139fd0cb51b24f97e6715394c3aa484ac601628204d47c057b11d448f15c5d1bf1e90760d10f
|
7
|
+
data.tar.gz: 245ada14810d09437af36e44f02898129f259902281a10cf0b870c6654ad1525c1b74d4f34138cdcfdf1920c4602a4880cc78194d18a8c5912db67d386f1ab99
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,46 @@
|
|
1
1
|
Unreleased Changes
|
2
2
|
------------------
|
3
3
|
|
4
|
+
1.198.0 (2025-08-26)
|
5
|
+
------------------
|
6
|
+
|
7
|
+
* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
|
8
|
+
|
9
|
+
* Issue - Fix multipart `download_file` to support `Pathname`, `File` and `Tempfile` objects as download destinations.
|
10
|
+
|
11
|
+
1.197.0 (2025-08-19)
|
12
|
+
------------------
|
13
|
+
|
14
|
+
* Issue - When multipart stream uploader fails to complete multipart upload, it calls abort multipart upload.
|
15
|
+
|
16
|
+
* Issue - For `Aws::S3::Object` class, the following methods have been deprecated: `download_file`, `upload_file` and `upload_stream`. Use `Aws::S3::TransferManager` instead.
|
17
|
+
|
18
|
+
* Feature - Add `Aws::S3::TransferManager`, a S3 transfer utility that provides upload/download capabilities with automatic multipart handling, progress tracking, and handling of large files.
|
19
|
+
|
20
|
+
1.196.1 (2025-08-05)
|
21
|
+
------------------
|
22
|
+
|
23
|
+
* Issue - Add range validation to multipart download to ensure all parts are successfully processed.
|
24
|
+
|
25
|
+
* Issue - When multipart uploader fails to complete multipart upload, it calls abort multipart upload.
|
26
|
+
|
27
|
+
* Issue - Clean up partially downloaded file on multipart `download_file` failure while preserving existing file.
|
28
|
+
|
29
|
+
1.196.0 (2025-08-04)
|
30
|
+
------------------
|
31
|
+
|
32
|
+
* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
|
33
|
+
|
34
|
+
1.195.0 (2025-07-31)
|
35
|
+
------------------
|
36
|
+
|
37
|
+
* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
|
38
|
+
|
39
|
+
1.194.0 (2025-07-21)
|
40
|
+
------------------
|
41
|
+
|
42
|
+
* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
|
43
|
+
|
4
44
|
1.193.0 (2025-07-15)
|
5
45
|
------------------
|
6
46
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.198.0
|
data/lib/aws-sdk-s3/client.rb
CHANGED
@@ -137,8 +137,8 @@ module Aws::S3
|
|
137
137
|
# class name or an instance of a plugin class.
|
138
138
|
#
|
139
139
|
# @option options [required, Aws::CredentialProvider] :credentials
|
140
|
-
# Your AWS credentials. This can be
|
141
|
-
# following classes:
|
140
|
+
# Your AWS credentials used for authentication. This can be any class that includes and implements
|
141
|
+
# `Aws::CredentialProvider`, or instance of any one of the following classes:
|
142
142
|
#
|
143
143
|
# * `Aws::Credentials` - Used for configuring static, non-refreshing
|
144
144
|
# credentials.
|
@@ -166,22 +166,24 @@ module Aws::S3
|
|
166
166
|
# * `Aws::CognitoIdentityCredentials` - Used for loading credentials
|
167
167
|
# from the Cognito Identity service.
|
168
168
|
#
|
169
|
-
# When `:credentials` are not configured directly, the following
|
170
|
-
# locations will be searched for credentials:
|
169
|
+
# When `:credentials` are not configured directly, the following locations will be searched for credentials:
|
171
170
|
#
|
172
171
|
# * `Aws.config[:credentials]`
|
172
|
+
#
|
173
173
|
# * The `:access_key_id`, `:secret_access_key`, `:session_token`, and
|
174
174
|
# `:account_id` options.
|
175
|
-
#
|
176
|
-
#
|
175
|
+
#
|
176
|
+
# * `ENV['AWS_ACCESS_KEY_ID']`, `ENV['AWS_SECRET_ACCESS_KEY']`,
|
177
|
+
# `ENV['AWS_SESSION_TOKEN']`, and `ENV['AWS_ACCOUNT_ID']`.
|
178
|
+
#
|
177
179
|
# * `~/.aws/credentials`
|
180
|
+
#
|
178
181
|
# * `~/.aws/config`
|
179
|
-
#
|
180
|
-
#
|
181
|
-
# `Aws::InstanceProfileCredentials` or `Aws::ECSCredentials` to
|
182
|
-
# enable retries and extended timeouts. Instance profile credential
|
183
|
-
#
|
184
|
-
# to true.
|
182
|
+
#
|
183
|
+
# * EC2/ECS IMDS instance profile - When used by default, the timeouts are very aggressive.
|
184
|
+
# Construct and pass an instance of `Aws::InstanceProfileCredentials` or `Aws::ECSCredentials` to
|
185
|
+
# enable retries and extended timeouts. Instance profile credential fetching can be disabled by
|
186
|
+
# setting `ENV['AWS_EC2_METADATA_DISABLED']` to `true`.
|
185
187
|
#
|
186
188
|
# @option options [required, String] :region
|
187
189
|
# The AWS region to connect to. The configured `:region` is
|
@@ -219,6 +221,11 @@ module Aws::S3
|
|
219
221
|
# When false, the request will raise a `RetryCapacityNotAvailableError` and will
|
220
222
|
# not retry instead of sleeping.
|
221
223
|
#
|
224
|
+
# @option options [Array<String>] :auth_scheme_preference
|
225
|
+
# A list of preferred authentication schemes to use when making a request. Supported values are:
|
226
|
+
# `sigv4`, `sigv4a`, `httpBearerAuth`, and `noAuth`. When set using `ENV['AWS_AUTH_SCHEME_PREFERENCE']` or in
|
227
|
+
# shared config as `auth_scheme_preference`, the value should be a comma-separated list.
|
228
|
+
#
|
222
229
|
# @option options [Boolean] :client_side_monitoring (false)
|
223
230
|
# When `true`, client-side metrics will be collected for all API requests from
|
224
231
|
# this client.
|
@@ -333,8 +340,8 @@ module Aws::S3
|
|
333
340
|
# When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way.
|
334
341
|
#
|
335
342
|
# @option options [String] :profile ("default")
|
336
|
-
# Used when loading credentials from the shared credentials file
|
337
|
-
#
|
343
|
+
# Used when loading credentials from the shared credentials file at `HOME/.aws/credentials`.
|
344
|
+
# When not specified, 'default' is used.
|
338
345
|
#
|
339
346
|
# @option options [String] :request_checksum_calculation ("when_supported")
|
340
347
|
# Determines when a checksum will be calculated for request payloads. Values are:
|
@@ -466,8 +473,8 @@ module Aws::S3
|
|
466
473
|
# `Aws::Telemetry::OTelProvider` for telemetry provider.
|
467
474
|
#
|
468
475
|
# @option options [Aws::TokenProvider] :token_provider
|
469
|
-
#
|
470
|
-
# following classes:
|
476
|
+
# Your Bearer token used for authentication. This can be any class that includes and implements
|
477
|
+
# `Aws::TokenProvider`, or instance of any one of the following classes:
|
471
478
|
#
|
472
479
|
# * `Aws::StaticTokenProvider` - Used for configuring static, non-refreshing
|
473
480
|
# tokens.
|
@@ -21728,7 +21735,7 @@ module Aws::S3
|
|
21728
21735
|
tracer: tracer
|
21729
21736
|
)
|
21730
21737
|
context[:gem_name] = 'aws-sdk-s3'
|
21731
|
-
context[:gem_version] = '1.
|
21738
|
+
context[:gem_version] = '1.198.0'
|
21732
21739
|
Seahorse::Client::Request.new(handlers, context)
|
21733
21740
|
end
|
21734
21741
|
|
@@ -398,14 +398,14 @@ module Aws
|
|
398
398
|
end
|
399
399
|
true
|
400
400
|
end
|
401
|
+
deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
|
401
402
|
|
402
403
|
# Uploads a file from disk to the current object in S3.
|
403
404
|
#
|
404
405
|
# # small files are uploaded in a single API call
|
405
406
|
# obj.upload_file('/path/to/file')
|
406
407
|
#
|
407
|
-
# Files larger than or equal to `:multipart_threshold` are uploaded
|
408
|
-
# using the Amazon S3 multipart upload APIs.
|
408
|
+
# Files larger than or equal to `:multipart_threshold` are uploaded using the Amazon S3 multipart upload APIs.
|
409
409
|
#
|
410
410
|
# # large files are automatically split into parts
|
411
411
|
# # and the parts are uploaded in parallel
|
@@ -421,47 +421,37 @@ module Aws
|
|
421
421
|
# You can provide a callback to monitor progress of the upload:
|
422
422
|
#
|
423
423
|
# # bytes and totals are each an array with 1 entry per part
|
424
|
-
# progress =
|
425
|
-
# puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
|
424
|
+
# progress = proc do |bytes, totals|
|
425
|
+
# puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
|
426
426
|
# end
|
427
427
|
# obj.upload_file('/path/to/file', progress_callback: progress)
|
428
428
|
#
|
429
|
-
# @param [String, Pathname, File, Tempfile] source A file on the local
|
430
|
-
#
|
431
|
-
#
|
432
|
-
#
|
433
|
-
# you are responsible for closing it after the upload completes. When
|
434
|
-
# using an open Tempfile, rewind it before uploading or else the object
|
429
|
+
# @param [String, Pathname, File, Tempfile] source A file on the local file system that will be uploaded as
|
430
|
+
# this object. This can either be a String or Pathname to the file, an open File object, or an open
|
431
|
+
# Tempfile object. If you pass an open File or Tempfile object, then you are responsible for closing it
|
432
|
+
# after the upload completes. When using an open Tempfile, rewind it before uploading or else the object
|
435
433
|
# will be empty.
|
436
434
|
#
|
437
435
|
# @param [Hash] options
|
438
|
-
# Additional options for {Client#put_object}
|
439
|
-
#
|
440
|
-
#
|
441
|
-
# {Client#complete_multipart_upload},
|
442
|
-
# and {Client#upload_part} can be provided.
|
436
|
+
# Additional options for {Client#put_object} when file sizes below the multipart threshold.
|
437
|
+
# For files larger than the multipart threshold, options for {Client#create_multipart_upload},
|
438
|
+
# {Client#complete_multipart_upload}, and {Client#upload_part} can be provided.
|
443
439
|
#
|
444
|
-
# @option options [Integer] :multipart_threshold (104857600) Files larger
|
445
|
-
#
|
446
|
-
# multipart APIs.
|
447
|
-
# Default threshold is 100MB.
|
440
|
+
# @option options [Integer] :multipart_threshold (104857600) Files larger han or equal to
|
441
|
+
# `:multipart_threshold` are uploaded using the S3 multipart APIs. Default threshold is 100MB.
|
448
442
|
#
|
449
|
-
# @option options [Integer] :thread_count (10) The number of parallel
|
450
|
-
#
|
451
|
-
# `:multipart_threshold`.
|
443
|
+
# @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
|
444
|
+
# This option is not used if the file is smaller than `:multipart_threshold`.
|
452
445
|
#
|
453
446
|
# @option options [Proc] :progress_callback
|
454
447
|
# A Proc that will be called when each chunk of the upload is sent.
|
455
448
|
# It will be invoked with [bytes_read], [total_sizes]
|
456
449
|
#
|
457
|
-
# @raise [MultipartUploadError] If an object is being uploaded in
|
458
|
-
#
|
459
|
-
#
|
460
|
-
# method that returns the failures that caused the upload to be
|
461
|
-
# aborted.
|
450
|
+
# @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed,
|
451
|
+
# then the upload is aborted and this error is raised. The raised error has a `#errors` method that
|
452
|
+
# returns the failures that caused the upload to be aborted.
|
462
453
|
#
|
463
|
-
# @return [Boolean] Returns `true` when the object is uploaded
|
464
|
-
# without any errors.
|
454
|
+
# @return [Boolean] Returns `true` when the object is uploaded without any errors.
|
465
455
|
#
|
466
456
|
# @see Client#put_object
|
467
457
|
# @see Client#create_multipart_upload
|
@@ -469,26 +459,21 @@ module Aws
|
|
469
459
|
# @see Client#upload_part
|
470
460
|
def upload_file(source, options = {})
|
471
461
|
uploading_options = options.dup
|
472
|
-
uploader = FileUploader.new(
|
473
|
-
multipart_threshold: uploading_options.delete(:multipart_threshold),
|
474
|
-
client: client
|
475
|
-
)
|
462
|
+
uploader = FileUploader.new(multipart_threshold: uploading_options.delete(:multipart_threshold), client: client)
|
476
463
|
response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
477
|
-
uploader.upload(
|
478
|
-
source,
|
479
|
-
uploading_options.merge(bucket: bucket_name, key: key)
|
480
|
-
)
|
464
|
+
uploader.upload(source, uploading_options.merge(bucket: bucket_name, key: key))
|
481
465
|
end
|
482
466
|
yield response if block_given?
|
483
467
|
true
|
484
468
|
end
|
469
|
+
deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
|
485
470
|
|
486
471
|
# Downloads a file in S3 to a path on disk.
|
487
472
|
#
|
488
473
|
# # small files (< 5MB) are downloaded in a single API call
|
489
474
|
# obj.download_file('/path/to/file')
|
490
475
|
#
|
491
|
-
# Files larger than 5MB are downloaded using multipart method
|
476
|
+
# Files larger than 5MB are downloaded using multipart method:
|
492
477
|
#
|
493
478
|
# # large files are split into parts
|
494
479
|
# # and the parts are downloaded in parallel
|
@@ -498,67 +483,63 @@ module Aws
|
|
498
483
|
#
|
499
484
|
# # bytes and part_sizes are each an array with 1 entry per part
|
500
485
|
# # part_sizes may not be known until the first bytes are retrieved
|
501
|
-
# progress =
|
502
|
-
# puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{part_sizes[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
|
486
|
+
# progress = proc do |bytes, part_sizes, file_size|
|
487
|
+
# puts bytes.map.with_index { |b, i| "Part #{i + 1}: #{b} / #{part_sizes[i]}" }.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
|
503
488
|
# end
|
504
489
|
# obj.download_file('/path/to/file', progress_callback: progress)
|
505
490
|
#
|
506
|
-
# @param [String
|
491
|
+
# @param [String, Pathname, File, Tempfile] destination
|
492
|
+
# Where to download the file to. This can either be a String or Pathname to the file, an open File object,
|
493
|
+
# or an open Tempfile object. If you pass an open File or Tempfile object, then you are responsible for
|
494
|
+
# closing it after the download completes.
|
507
495
|
#
|
508
496
|
# @param [Hash] options
|
509
|
-
# Additional options for {Client#get_object} and #{Client#head_object}
|
510
|
-
# may be provided.
|
497
|
+
# Additional options for {Client#get_object} and #{Client#head_object} may be provided.
|
511
498
|
#
|
512
|
-
# @option options [String] mode `auto`, `single_request
|
513
|
-
# `single_request` mode forces only 1 GET request is made in download,
|
514
|
-
# `get_range` mode allows `chunk_size` parameter to configured in
|
515
|
-
# customizing each range size in multipart_download,
|
516
|
-
# By default, `auto` mode is enabled, which performs multipart_download
|
499
|
+
# @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"`
|
517
500
|
#
|
518
|
-
#
|
501
|
+
# * `auto` mode is enabled by default, which performs `multipart_download`
|
502
|
+
# * `"single_request`" mode forces only 1 GET request is made in download
|
503
|
+
# * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size
|
519
504
|
#
|
520
|
-
# @option options [Integer]
|
521
|
-
# the multipart download.
|
505
|
+
# @option options [Integer] :chunk_size required in `"get_range"` mode.
|
522
506
|
#
|
523
|
-
# @option options [
|
524
|
-
# retrieve the object. For more about object versioning, see:
|
525
|
-
# https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
|
507
|
+
# @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
|
526
508
|
#
|
527
|
-
# @option options [String]
|
528
|
-
# the object has a stored checksum, it will be used to validate the
|
529
|
-
# download and will raise an `Aws::Errors::ChecksumError` if
|
530
|
-
# checksum validation fails. You may provide a `on_checksum_validated`
|
531
|
-
# callback if you need to verify that validation occurred and which
|
532
|
-
# algorithm was used. To disable checksum validation, set
|
533
|
-
# `checksum_mode` to "DISABLED".
|
509
|
+
# @option options [String] :version_id The object version id used to retrieve the object.
|
534
510
|
#
|
535
|
-
#
|
536
|
-
#
|
537
|
-
#
|
538
|
-
#
|
511
|
+
# @see https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html ObjectVersioning
|
512
|
+
#
|
513
|
+
# @option options [String] :checksum_mode ("ENABLED")
|
514
|
+
# When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will
|
515
|
+
# raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated`
|
516
|
+
# callback if you need to verify that validation occurred and which algorithm was used.
|
517
|
+
# To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
|
518
|
+
#
|
519
|
+
# @option options [Callable] :on_checksum_validated
|
520
|
+
# Called each time a request's checksum is validated with the checksum algorithm and the
|
521
|
+
# response. For multipart downloads, this will be called for each part that is downloaded and validated.
|
539
522
|
#
|
540
523
|
# @option options [Proc] :progress_callback
|
541
|
-
# A Proc that will be called when each chunk of the download is received.
|
542
|
-
#
|
543
|
-
#
|
544
|
-
#
|
545
|
-
# callback until the first bytes in the part are received.
|
524
|
+
# A Proc that will be called when each chunk of the download is received. It will be invoked with
|
525
|
+
# `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges),
|
526
|
+
# the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes
|
527
|
+
# in the part are received.
|
546
528
|
#
|
547
|
-
# @
|
548
|
-
#
|
529
|
+
# @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors.
|
530
|
+
#
|
531
|
+
# @return [Boolean] Returns `true` when the file is downloaded without any errors.
|
549
532
|
#
|
550
533
|
# @see Client#get_object
|
551
534
|
# @see Client#head_object
|
552
535
|
def download_file(destination, options = {})
|
553
536
|
downloader = FileDownloader.new(client: client)
|
554
537
|
Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
|
555
|
-
downloader.download(
|
556
|
-
destination,
|
557
|
-
options.merge(bucket: bucket_name, key: key)
|
558
|
-
)
|
538
|
+
downloader.download(destination, options.merge(bucket: bucket_name, key: key))
|
559
539
|
end
|
560
540
|
true
|
561
541
|
end
|
542
|
+
deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
|
562
543
|
|
563
544
|
class Collection < Aws::Resources::Collection
|
564
545
|
alias_method :delete, :batch_delete!
|
@@ -10,6 +10,7 @@ module Aws
|
|
10
10
|
autoload :FileUploader, 'aws-sdk-s3/file_uploader'
|
11
11
|
autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
|
12
12
|
autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
|
13
|
+
autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
|
13
14
|
autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
|
14
15
|
autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
|
15
16
|
autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
|
@@ -17,13 +18,13 @@ module Aws
|
|
17
18
|
autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
|
18
19
|
autoload :PresignedPost, 'aws-sdk-s3/presigned_post'
|
19
20
|
autoload :Presigner, 'aws-sdk-s3/presigner'
|
21
|
+
autoload :TransferManager, 'aws-sdk-s3/transfer_manager'
|
20
22
|
|
21
23
|
# s3 express session auth
|
22
24
|
autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials'
|
23
25
|
autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider'
|
24
26
|
|
25
27
|
# s3 access grants auth
|
26
|
-
|
27
28
|
autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials'
|
28
29
|
autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider'
|
29
30
|
end
|
@@ -1,9 +1,8 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require 'pathname'
|
4
|
-
require '
|
4
|
+
require 'securerandom'
|
5
5
|
require 'set'
|
6
|
-
require 'tmpdir'
|
7
6
|
|
8
7
|
module Aws
|
9
8
|
module S3
|
@@ -12,7 +11,6 @@ module Aws
|
|
12
11
|
|
13
12
|
MIN_CHUNK_SIZE = 5 * 1024 * 1024
|
14
13
|
MAX_PARTS = 10_000
|
15
|
-
THREAD_COUNT = 10
|
16
14
|
|
17
15
|
def initialize(options = {})
|
18
16
|
@client = options[:client] || Client.new
|
@@ -22,18 +20,18 @@ module Aws
|
|
22
20
|
attr_reader :client
|
23
21
|
|
24
22
|
def download(destination, options = {})
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
@
|
34
|
-
@on_checksum_validated = options
|
35
|
-
@progress_callback = options
|
36
|
-
|
23
|
+
valid_types = [String, Pathname, File, Tempfile]
|
24
|
+
unless valid_types.include?(destination.class)
|
25
|
+
raise ArgumentError, "Invalid destination, expected #{valid_types.join(', ')} but got: #{destination.class}"
|
26
|
+
end
|
27
|
+
|
28
|
+
@destination = destination
|
29
|
+
@mode = options.delete(:mode) || 'auto'
|
30
|
+
@thread_count = options.delete(:thread_count) || 10
|
31
|
+
@chunk_size = options.delete(:chunk_size)
|
32
|
+
@on_checksum_validated = options.delete(:on_checksum_validated)
|
33
|
+
@progress_callback = options.delete(:progress_callback)
|
34
|
+
@params = options
|
37
35
|
validate!
|
38
36
|
|
39
37
|
Aws::Plugins::UserAgent.metric('S3_TRANSFER') do
|
@@ -41,32 +39,31 @@ module Aws
|
|
41
39
|
when 'auto' then multipart_download
|
42
40
|
when 'single_request' then single_request
|
43
41
|
when 'get_range'
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
msg = 'In :get_range mode, :chunk_size must be provided'
|
49
|
-
raise ArgumentError, msg
|
50
|
-
end
|
42
|
+
raise ArgumentError, 'In get_range mode, :chunk_size must be provided' unless @chunk_size
|
43
|
+
|
44
|
+
resp = @client.head_object(@params)
|
45
|
+
multithreaded_get_by_ranges(resp.content_length, resp.etag)
|
51
46
|
else
|
52
|
-
|
53
|
-
'mode should be :single_request, :get_range or :auto'
|
54
|
-
raise ArgumentError, msg
|
47
|
+
raise ArgumentError, "Invalid mode #{@mode} provided, :mode should be single_request, get_range or auto"
|
55
48
|
end
|
56
49
|
end
|
50
|
+
File.rename(@temp_path, @destination) if @temp_path
|
51
|
+
ensure
|
52
|
+
File.delete(@temp_path) if @temp_path && File.exist?(@temp_path)
|
57
53
|
end
|
58
54
|
|
59
55
|
private
|
60
56
|
|
61
57
|
def validate!
|
62
|
-
|
63
|
-
|
64
|
-
|
58
|
+
return unless @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
|
59
|
+
|
60
|
+
raise ArgumentError, ':on_checksum_validated must be callable'
|
65
61
|
end
|
66
62
|
|
67
63
|
def multipart_download
|
68
64
|
resp = @client.head_object(@params.merge(part_number: 1))
|
69
65
|
count = resp.parts_count
|
66
|
+
|
70
67
|
if count.nil? || count <= 1
|
71
68
|
if resp.content_length <= MIN_CHUNK_SIZE
|
72
69
|
single_request
|
@@ -74,8 +71,8 @@ module Aws
|
|
74
71
|
multithreaded_get_by_ranges(resp.content_length, resp.etag)
|
75
72
|
end
|
76
73
|
else
|
77
|
-
#
|
78
|
-
resp = @client.head_object(@params)
|
74
|
+
# covers cases when given object is not uploaded via UploadPart API
|
75
|
+
resp = @client.head_object(@params) # partNumber is an option
|
79
76
|
if resp.content_length <= MIN_CHUNK_SIZE
|
80
77
|
single_request
|
81
78
|
else
|
@@ -86,7 +83,7 @@ module Aws
|
|
86
83
|
|
87
84
|
def compute_mode(file_size, count, etag)
|
88
85
|
chunk_size = compute_chunk(file_size)
|
89
|
-
part_size = (file_size.to_f / count
|
86
|
+
part_size = (file_size.to_f / count).ceil
|
90
87
|
if chunk_size < part_size
|
91
88
|
multithreaded_get_by_ranges(file_size, etag)
|
92
89
|
else
|
@@ -94,32 +91,10 @@ module Aws
|
|
94
91
|
end
|
95
92
|
end
|
96
93
|
|
97
|
-
def construct_chunks(file_size)
|
98
|
-
offset = 0
|
99
|
-
default_chunk_size = compute_chunk(file_size)
|
100
|
-
chunks = []
|
101
|
-
while offset < file_size
|
102
|
-
progress = offset + default_chunk_size
|
103
|
-
progress = file_size if progress > file_size
|
104
|
-
chunks << "bytes=#{offset}-#{progress - 1}"
|
105
|
-
offset = progress
|
106
|
-
end
|
107
|
-
chunks
|
108
|
-
end
|
109
|
-
|
110
94
|
def compute_chunk(file_size)
|
111
|
-
if @chunk_size && @chunk_size > file_size
|
112
|
-
raise ArgumentError, ":chunk_size shouldn't exceed total file size."
|
113
|
-
else
|
114
|
-
@chunk_size || [
|
115
|
-
(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE
|
116
|
-
].max.to_i
|
117
|
-
end
|
118
|
-
end
|
95
|
+
raise ArgumentError, ":chunk_size shouldn't exceed total file size." if @chunk_size && @chunk_size > file_size
|
119
96
|
|
120
|
-
|
121
|
-
chunks = (1..chunks) if mode.eql? 'part_number'
|
122
|
-
chunks.each_slice(@thread_count).to_a
|
97
|
+
@chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
|
123
98
|
end
|
124
99
|
|
125
100
|
def multithreaded_get_by_ranges(file_size, etag)
|
@@ -130,12 +105,8 @@ module Aws
|
|
130
105
|
while offset < file_size
|
131
106
|
progress = offset + default_chunk_size
|
132
107
|
progress = file_size if progress > file_size
|
133
|
-
|
134
|
-
chunks << Part.new(
|
135
|
-
part_number: part_number,
|
136
|
-
size: (progress-offset),
|
137
|
-
params: @params.merge(range: range, if_match: etag)
|
138
|
-
)
|
108
|
+
params = @params.merge(range: "bytes=#{offset}-#{progress - 1}", if_match: etag)
|
109
|
+
chunks << Part.new(part_number: part_number, size: (progress - offset), params: params)
|
139
110
|
part_number += 1
|
140
111
|
offset = progress
|
141
112
|
end
|
@@ -152,10 +123,13 @@ module Aws
|
|
152
123
|
def download_in_threads(pending, total_size)
|
153
124
|
threads = []
|
154
125
|
progress = MultipartProgress.new(pending, total_size, @progress_callback) if @progress_callback
|
126
|
+
unless [File, Tempfile].include?(@destination.class)
|
127
|
+
@temp_path = "#{@destination}.s3tmp.#{SecureRandom.alphanumeric(8)}"
|
128
|
+
end
|
155
129
|
@thread_count.times do
|
156
130
|
thread = Thread.new do
|
157
131
|
begin
|
158
|
-
while part = pending.shift
|
132
|
+
while (part = pending.shift)
|
159
133
|
if progress
|
160
134
|
part.params[:on_chunk_received] =
|
161
135
|
proc do |_chunk, bytes, total|
|
@@ -163,16 +137,17 @@ module Aws
|
|
163
137
|
end
|
164
138
|
end
|
165
139
|
resp = @client.get_object(part.params)
|
166
|
-
|
140
|
+
range = extract_range(resp.content_range)
|
141
|
+
validate_range(range, part.params[:range]) if part.params[:range]
|
142
|
+
write(resp.body, range)
|
167
143
|
if @on_checksum_validated && resp.checksum_validated
|
168
144
|
@on_checksum_validated.call(resp.checksum_validated, resp)
|
169
145
|
end
|
170
146
|
end
|
171
147
|
nil
|
172
|
-
rescue =>
|
173
|
-
# keep other threads from downloading other parts
|
174
|
-
|
175
|
-
raise error
|
148
|
+
rescue StandardError => e
|
149
|
+
pending.clear! # keep other threads from downloading other parts
|
150
|
+
raise e
|
176
151
|
end
|
177
152
|
end
|
178
153
|
threads << thread
|
@@ -180,21 +155,28 @@ module Aws
|
|
180
155
|
threads.map(&:value).compact
|
181
156
|
end
|
182
157
|
|
183
|
-
def
|
184
|
-
|
185
|
-
|
186
|
-
|
158
|
+
def extract_range(value)
|
159
|
+
value.match(%r{bytes (?<range>\d+-\d+)/\d+})[:range]
|
160
|
+
end
|
161
|
+
|
162
|
+
def validate_range(actual, expected)
|
163
|
+
return if actual == expected.match(/bytes=(?<range>\d+-\d+)/)[:range]
|
164
|
+
|
165
|
+
raise MultipartDownloadError, "multipart download failed: expected range of #{expected} but got #{actual}"
|
166
|
+
end
|
167
|
+
|
168
|
+
def write(body, range)
|
169
|
+
path = @temp_path || @destination
|
170
|
+
File.write(path, body.read, range.split('-').first.to_i)
|
187
171
|
end
|
188
172
|
|
189
173
|
def single_request
|
190
|
-
params = @params.merge(response_target: @
|
174
|
+
params = @params.merge(response_target: @destination)
|
191
175
|
params[:on_chunk_received] = single_part_progress if @progress_callback
|
192
176
|
resp = @client.get_object(params)
|
193
|
-
|
194
177
|
return resp unless @on_checksum_validated
|
195
178
|
|
196
179
|
@on_checksum_validated.call(resp.checksum_validated, resp) if resp.checksum_validated
|
197
|
-
|
198
180
|
resp
|
199
181
|
end
|
200
182
|
|
@@ -204,6 +186,7 @@ module Aws
|
|
204
186
|
end
|
205
187
|
end
|
206
188
|
|
189
|
+
# @api private
|
207
190
|
class Part < Struct.new(:part_number, :size, :params)
|
208
191
|
include Aws::Structure
|
209
192
|
end
|