aws-sdk-s3 1.193.0 → 1.198.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d0e94777097f81374f7ae8c0f3bd8c76a6ab248137104477cef54bfbb5f88ae0
4
- data.tar.gz: 2d7dac3b84fcba84b3bcc125e75d95183eb129ee7b1728ea6ca579f575cc2061
3
+ metadata.gz: 8f8abf2a326b6ebef3c9c9ceac08c2378c4e0d9bc3e187a4a3dd89bb81fb8154
4
+ data.tar.gz: 6186fe037f7fc5541c2a8e1006c9cb0a34630c69fb4a0bdebc40a63b872c9189
5
5
  SHA512:
6
- metadata.gz: 197d9315a2d83377a7870741c01d7e417fc23080bb20e947a5063748e334b07cd268d974129b95e8cc1e3ee7d127596c2e0040e204868cae00f4e8fc9af36f2c
7
- data.tar.gz: eb4a3d2fb4570da19d661649f105bb6697392b45a88779178d786595f127c1a6d3d02bc8a56ab128f50a86afb640d5c22836dcc044d48871ab968fa26dff2c77
6
+ metadata.gz: 5aade0fd0688cee874c30704211d6f166ea8aaca4838adcdd2ad139fd0cb51b24f97e6715394c3aa484ac601628204d47c057b11d448f15c5d1bf1e90760d10f
7
+ data.tar.gz: 245ada14810d09437af36e44f02898129f259902281a10cf0b870c6654ad1525c1b74d4f34138cdcfdf1920c4602a4880cc78194d18a8c5912db67d386f1ab99
data/CHANGELOG.md CHANGED
@@ -1,6 +1,46 @@
1
1
  Unreleased Changes
2
2
  ------------------
3
3
 
4
+ 1.198.0 (2025-08-26)
5
+ ------------------
6
+
7
+ * Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
8
+
9
+ * Issue - Fix multipart `download_file` to support `Pathname`, `File` and `Tempfile` objects as download destinations.
10
+
11
+ 1.197.0 (2025-08-19)
12
+ ------------------
13
+
14
+ * Issue - When multipart stream uploader fails to complete multipart upload, it calls abort multipart upload.
15
+
16
+ * Issue - For `Aws::S3::Object` class, the following methods have been deprecated: `download_file`, `upload_file` and `upload_stream`. Use `Aws::S3::TransferManager` instead.
17
+
18
+ * Feature - Add `Aws::S3::TransferManager`, a S3 transfer utility that provides upload/download capabilities with automatic multipart handling, progress tracking, and handling of large files.
19
+
20
+ 1.196.1 (2025-08-05)
21
+ ------------------
22
+
23
+ * Issue - Add range validation to multipart download to ensure all parts are successfully processed.
24
+
25
+ * Issue - When multipart uploader fails to complete multipart upload, it calls abort multipart upload.
26
+
27
+ * Issue - Clean up partially downloaded file on multipart `download_file` failure while preserving existing file.
28
+
29
+ 1.196.0 (2025-08-04)
30
+ ------------------
31
+
32
+ * Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
33
+
34
+ 1.195.0 (2025-07-31)
35
+ ------------------
36
+
37
+ * Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
38
+
39
+ 1.194.0 (2025-07-21)
40
+ ------------------
41
+
42
+ * Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
43
+
4
44
  1.193.0 (2025-07-15)
5
45
  ------------------
6
46
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 1.193.0
1
+ 1.198.0
@@ -137,8 +137,8 @@ module Aws::S3
137
137
  # class name or an instance of a plugin class.
138
138
  #
139
139
  # @option options [required, Aws::CredentialProvider] :credentials
140
- # Your AWS credentials. This can be an instance of any one of the
141
- # following classes:
140
+ # Your AWS credentials used for authentication. This can be any class that includes and implements
141
+ # `Aws::CredentialProvider`, or instance of any one of the following classes:
142
142
  #
143
143
  # * `Aws::Credentials` - Used for configuring static, non-refreshing
144
144
  # credentials.
@@ -166,22 +166,24 @@ module Aws::S3
166
166
  # * `Aws::CognitoIdentityCredentials` - Used for loading credentials
167
167
  # from the Cognito Identity service.
168
168
  #
169
- # When `:credentials` are not configured directly, the following
170
- # locations will be searched for credentials:
169
+ # When `:credentials` are not configured directly, the following locations will be searched for credentials:
171
170
  #
172
171
  # * `Aws.config[:credentials]`
172
+ #
173
173
  # * The `:access_key_id`, `:secret_access_key`, `:session_token`, and
174
174
  # `:account_id` options.
175
- # * ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'],
176
- # ENV['AWS_SESSION_TOKEN'], and ENV['AWS_ACCOUNT_ID']
175
+ #
176
+ # * `ENV['AWS_ACCESS_KEY_ID']`, `ENV['AWS_SECRET_ACCESS_KEY']`,
177
+ # `ENV['AWS_SESSION_TOKEN']`, and `ENV['AWS_ACCOUNT_ID']`.
178
+ #
177
179
  # * `~/.aws/credentials`
180
+ #
178
181
  # * `~/.aws/config`
179
- # * EC2/ECS IMDS instance profile - When used by default, the timeouts
180
- # are very aggressive. Construct and pass an instance of
181
- # `Aws::InstanceProfileCredentials` or `Aws::ECSCredentials` to
182
- # enable retries and extended timeouts. Instance profile credential
183
- # fetching can be disabled by setting ENV['AWS_EC2_METADATA_DISABLED']
184
- # to true.
182
+ #
183
+ # * EC2/ECS IMDS instance profile - When used by default, the timeouts are very aggressive.
184
+ # Construct and pass an instance of `Aws::InstanceProfileCredentials` or `Aws::ECSCredentials` to
185
+ # enable retries and extended timeouts. Instance profile credential fetching can be disabled by
186
+ # setting `ENV['AWS_EC2_METADATA_DISABLED']` to `true`.
185
187
  #
186
188
  # @option options [required, String] :region
187
189
  # The AWS region to connect to. The configured `:region` is
@@ -219,6 +221,11 @@ module Aws::S3
219
221
  # When false, the request will raise a `RetryCapacityNotAvailableError` and will
220
222
  # not retry instead of sleeping.
221
223
  #
224
+ # @option options [Array<String>] :auth_scheme_preference
225
+ # A list of preferred authentication schemes to use when making a request. Supported values are:
226
+ # `sigv4`, `sigv4a`, `httpBearerAuth`, and `noAuth`. When set using `ENV['AWS_AUTH_SCHEME_PREFERENCE']` or in
227
+ # shared config as `auth_scheme_preference`, the value should be a comma-separated list.
228
+ #
222
229
  # @option options [Boolean] :client_side_monitoring (false)
223
230
  # When `true`, client-side metrics will be collected for all API requests from
224
231
  # this client.
@@ -333,8 +340,8 @@ module Aws::S3
333
340
  # When an EventStream or Proc object is provided, it will be used as callback for each chunk of event stream response received along the way.
334
341
  #
335
342
  # @option options [String] :profile ("default")
336
- # Used when loading credentials from the shared credentials file
337
- # at HOME/.aws/credentials. When not specified, 'default' is used.
343
+ # Used when loading credentials from the shared credentials file at `HOME/.aws/credentials`.
344
+ # When not specified, 'default' is used.
338
345
  #
339
346
  # @option options [String] :request_checksum_calculation ("when_supported")
340
347
  # Determines when a checksum will be calculated for request payloads. Values are:
@@ -466,8 +473,8 @@ module Aws::S3
466
473
  # `Aws::Telemetry::OTelProvider` for telemetry provider.
467
474
  #
468
475
  # @option options [Aws::TokenProvider] :token_provider
469
- # A Bearer Token Provider. This can be an instance of any one of the
470
- # following classes:
476
+ # Your Bearer token used for authentication. This can be any class that includes and implements
477
+ # `Aws::TokenProvider`, or instance of any one of the following classes:
471
478
  #
472
479
  # * `Aws::StaticTokenProvider` - Used for configuring static, non-refreshing
473
480
  # tokens.
@@ -21728,7 +21735,7 @@ module Aws::S3
21728
21735
  tracer: tracer
21729
21736
  )
21730
21737
  context[:gem_name] = 'aws-sdk-s3'
21731
- context[:gem_version] = '1.193.0'
21738
+ context[:gem_version] = '1.198.0'
21732
21739
  Seahorse::Client::Request.new(handlers, context)
21733
21740
  end
21734
21741
 
@@ -398,14 +398,14 @@ module Aws
398
398
  end
399
399
  true
400
400
  end
401
+ deprecated(:upload_stream, use: 'Aws::S3::TransferManager#upload_stream', version: 'next major version')
401
402
 
402
403
  # Uploads a file from disk to the current object in S3.
403
404
  #
404
405
  # # small files are uploaded in a single API call
405
406
  # obj.upload_file('/path/to/file')
406
407
  #
407
- # Files larger than or equal to `:multipart_threshold` are uploaded
408
- # using the Amazon S3 multipart upload APIs.
408
+ # Files larger than or equal to `:multipart_threshold` are uploaded using the Amazon S3 multipart upload APIs.
409
409
  #
410
410
  # # large files are automatically split into parts
411
411
  # # and the parts are uploaded in parallel
@@ -421,47 +421,37 @@ module Aws
421
421
  # You can provide a callback to monitor progress of the upload:
422
422
  #
423
423
  # # bytes and totals are each an array with 1 entry per part
424
- # progress = Proc.new do |bytes, totals|
425
- # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%" }
424
+ # progress = proc do |bytes, totals|
425
+ # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{totals[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / totals.sum }%"
426
426
  # end
427
427
  # obj.upload_file('/path/to/file', progress_callback: progress)
428
428
  #
429
- # @param [String, Pathname, File, Tempfile] source A file on the local
430
- # file system that will be uploaded as this object. This can either be
431
- # a String or Pathname to the file, an open File object, or an open
432
- # Tempfile object. If you pass an open File or Tempfile object, then
433
- # you are responsible for closing it after the upload completes. When
434
- # using an open Tempfile, rewind it before uploading or else the object
429
+ # @param [String, Pathname, File, Tempfile] source A file on the local file system that will be uploaded as
430
+ # this object. This can either be a String or Pathname to the file, an open File object, or an open
431
+ # Tempfile object. If you pass an open File or Tempfile object, then you are responsible for closing it
432
+ # after the upload completes. When using an open Tempfile, rewind it before uploading or else the object
435
433
  # will be empty.
436
434
  #
437
435
  # @param [Hash] options
438
- # Additional options for {Client#put_object}
439
- # when file sizes below the multipart threshold. For files larger than
440
- # the multipart threshold, options for {Client#create_multipart_upload},
441
- # {Client#complete_multipart_upload},
442
- # and {Client#upload_part} can be provided.
436
+ # Additional options for {Client#put_object} when file sizes below the multipart threshold.
437
+ # For files larger than the multipart threshold, options for {Client#create_multipart_upload},
438
+ # {Client#complete_multipart_upload}, and {Client#upload_part} can be provided.
443
439
  #
444
- # @option options [Integer] :multipart_threshold (104857600) Files larger
445
- # than or equal to `:multipart_threshold` are uploaded using the S3
446
- # multipart APIs.
447
- # Default threshold is 100MB.
440
+ # @option options [Integer] :multipart_threshold (104857600) Files larger han or equal to
441
+ # `:multipart_threshold` are uploaded using the S3 multipart APIs. Default threshold is 100MB.
448
442
  #
449
- # @option options [Integer] :thread_count (10) The number of parallel
450
- # multipart uploads. This option is not used if the file is smaller than
451
- # `:multipart_threshold`.
443
+ # @option options [Integer] :thread_count (10) The number of parallel multipart uploads.
444
+ # This option is not used if the file is smaller than `:multipart_threshold`.
452
445
  #
453
446
  # @option options [Proc] :progress_callback
454
447
  # A Proc that will be called when each chunk of the upload is sent.
455
448
  # It will be invoked with [bytes_read], [total_sizes]
456
449
  #
457
- # @raise [MultipartUploadError] If an object is being uploaded in
458
- # parts, and the upload can not be completed, then the upload is
459
- # aborted and this error is raised. The raised error has a `#errors`
460
- # method that returns the failures that caused the upload to be
461
- # aborted.
450
+ # @raise [MultipartUploadError] If an object is being uploaded in parts, and the upload can not be completed,
451
+ # then the upload is aborted and this error is raised. The raised error has a `#errors` method that
452
+ # returns the failures that caused the upload to be aborted.
462
453
  #
463
- # @return [Boolean] Returns `true` when the object is uploaded
464
- # without any errors.
454
+ # @return [Boolean] Returns `true` when the object is uploaded without any errors.
465
455
  #
466
456
  # @see Client#put_object
467
457
  # @see Client#create_multipart_upload
@@ -469,26 +459,21 @@ module Aws
469
459
  # @see Client#upload_part
470
460
  def upload_file(source, options = {})
471
461
  uploading_options = options.dup
472
- uploader = FileUploader.new(
473
- multipart_threshold: uploading_options.delete(:multipart_threshold),
474
- client: client
475
- )
462
+ uploader = FileUploader.new(multipart_threshold: uploading_options.delete(:multipart_threshold), client: client)
476
463
  response = Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
477
- uploader.upload(
478
- source,
479
- uploading_options.merge(bucket: bucket_name, key: key)
480
- )
464
+ uploader.upload(source, uploading_options.merge(bucket: bucket_name, key: key))
481
465
  end
482
466
  yield response if block_given?
483
467
  true
484
468
  end
469
+ deprecated(:upload_file, use: 'Aws::S3::TransferManager#upload_file', version: 'next major version')
485
470
 
486
471
  # Downloads a file in S3 to a path on disk.
487
472
  #
488
473
  # # small files (< 5MB) are downloaded in a single API call
489
474
  # obj.download_file('/path/to/file')
490
475
  #
491
- # Files larger than 5MB are downloaded using multipart method
476
+ # Files larger than 5MB are downloaded using multipart method:
492
477
  #
493
478
  # # large files are split into parts
494
479
  # # and the parts are downloaded in parallel
@@ -498,67 +483,63 @@ module Aws
498
483
  #
499
484
  # # bytes and part_sizes are each an array with 1 entry per part
500
485
  # # part_sizes may not be known until the first bytes are retrieved
501
- # progress = Proc.new do |bytes, part_sizes, file_size|
502
- # puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{part_sizes[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%" }
486
+ # progress = proc do |bytes, part_sizes, file_size|
487
+ # puts bytes.map.with_index { |b, i| "Part #{i + 1}: #{b} / #{part_sizes[i]}" }.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%"
503
488
  # end
504
489
  # obj.download_file('/path/to/file', progress_callback: progress)
505
490
  #
506
- # @param [String] destination Where to download the file to.
491
+ # @param [String, Pathname, File, Tempfile] destination
492
+ # Where to download the file to. This can either be a String or Pathname to the file, an open File object,
493
+ # or an open Tempfile object. If you pass an open File or Tempfile object, then you are responsible for
494
+ # closing it after the download completes.
507
495
  #
508
496
  # @param [Hash] options
509
- # Additional options for {Client#get_object} and #{Client#head_object}
510
- # may be provided.
497
+ # Additional options for {Client#get_object} and #{Client#head_object} may be provided.
511
498
  #
512
- # @option options [String] mode `auto`, `single_request`, `get_range`
513
- # `single_request` mode forces only 1 GET request is made in download,
514
- # `get_range` mode allows `chunk_size` parameter to configured in
515
- # customizing each range size in multipart_download,
516
- # By default, `auto` mode is enabled, which performs multipart_download
499
+ # @option options [String] :mode ("auto") `"auto"`, `"single_request"` or `"get_range"`
517
500
  #
518
- # @option options [Integer] chunk_size required in get_range mode.
501
+ # * `auto` mode is enabled by default, which performs `multipart_download`
502
+ # * `"single_request`" mode forces only 1 GET request is made in download
503
+ # * `"get_range"` mode requires `:chunk_size` parameter to configured in customizing each range size
519
504
  #
520
- # @option options [Integer] thread_count (10) Customize threads used in
521
- # the multipart download.
505
+ # @option options [Integer] :chunk_size required in `"get_range"` mode.
522
506
  #
523
- # @option options [String] version_id The object version id used to
524
- # retrieve the object. For more about object versioning, see:
525
- # https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
507
+ # @option options [Integer] :thread_count (10) Customize threads used in the multipart download.
526
508
  #
527
- # @option options [String] checksum_mode (ENABLED) When `ENABLED` and
528
- # the object has a stored checksum, it will be used to validate the
529
- # download and will raise an `Aws::Errors::ChecksumError` if
530
- # checksum validation fails. You may provide a `on_checksum_validated`
531
- # callback if you need to verify that validation occurred and which
532
- # algorithm was used. To disable checksum validation, set
533
- # `checksum_mode` to "DISABLED".
509
+ # @option options [String] :version_id The object version id used to retrieve the object.
534
510
  #
535
- # @option options [Callable] on_checksum_validated Called each time a
536
- # request's checksum is validated with the checksum algorithm and the
537
- # response. For multipart downloads, this will be called for each
538
- # part that is downloaded and validated.
511
+ # @see https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html ObjectVersioning
512
+ #
513
+ # @option options [String] :checksum_mode ("ENABLED")
514
+ # When `"ENABLED"` and the object has a stored checksum, it will be used to validate the download and will
515
+ # raise an `Aws::Errors::ChecksumError` if checksum validation fails. You may provide a `on_checksum_validated`
516
+ # callback if you need to verify that validation occurred and which algorithm was used.
517
+ # To disable checksum validation, set `checksum_mode` to `"DISABLED"`.
518
+ #
519
+ # @option options [Callable] :on_checksum_validated
520
+ # Called each time a request's checksum is validated with the checksum algorithm and the
521
+ # response. For multipart downloads, this will be called for each part that is downloaded and validated.
539
522
  #
540
523
  # @option options [Proc] :progress_callback
541
- # A Proc that will be called when each chunk of the download is received.
542
- # It will be invoked with [bytes_read], [part_sizes], file_size.
543
- # When the object is downloaded as parts (rather than by ranges), the
544
- # part_sizes will not be known ahead of time and will be nil in the
545
- # callback until the first bytes in the part are received.
524
+ # A Proc that will be called when each chunk of the download is received. It will be invoked with
525
+ # `bytes_read`, `part_sizes`, `file_size`. When the object is downloaded as parts (rather than by ranges),
526
+ # the `part_sizes` will not be known ahead of time and will be `nil` in the callback until the first bytes
527
+ # in the part are received.
546
528
  #
547
- # @return [Boolean] Returns `true` when the file is downloaded without
548
- # any errors.
529
+ # @raise [MultipartDownloadError] Raised when an object validation fails outside of service errors.
530
+ #
531
+ # @return [Boolean] Returns `true` when the file is downloaded without any errors.
549
532
  #
550
533
  # @see Client#get_object
551
534
  # @see Client#head_object
552
535
  def download_file(destination, options = {})
553
536
  downloader = FileDownloader.new(client: client)
554
537
  Aws::Plugins::UserAgent.metric('RESOURCE_MODEL') do
555
- downloader.download(
556
- destination,
557
- options.merge(bucket: bucket_name, key: key)
558
- )
538
+ downloader.download(destination, options.merge(bucket: bucket_name, key: key))
559
539
  end
560
540
  true
561
541
  end
542
+ deprecated(:download_file, use: 'Aws::S3::TransferManager#download_file', version: 'next major version')
562
543
 
563
544
  class Collection < Aws::Resources::Collection
564
545
  alias_method :delete, :batch_delete!
@@ -10,6 +10,7 @@ module Aws
10
10
  autoload :FileUploader, 'aws-sdk-s3/file_uploader'
11
11
  autoload :FileDownloader, 'aws-sdk-s3/file_downloader'
12
12
  autoload :LegacySigner, 'aws-sdk-s3/legacy_signer'
13
+ autoload :MultipartDownloadError, 'aws-sdk-s3/multipart_download_error'
13
14
  autoload :MultipartFileUploader, 'aws-sdk-s3/multipart_file_uploader'
14
15
  autoload :MultipartStreamUploader, 'aws-sdk-s3/multipart_stream_uploader'
15
16
  autoload :MultipartUploadError, 'aws-sdk-s3/multipart_upload_error'
@@ -17,13 +18,13 @@ module Aws
17
18
  autoload :ObjectMultipartCopier, 'aws-sdk-s3/object_multipart_copier'
18
19
  autoload :PresignedPost, 'aws-sdk-s3/presigned_post'
19
20
  autoload :Presigner, 'aws-sdk-s3/presigner'
21
+ autoload :TransferManager, 'aws-sdk-s3/transfer_manager'
20
22
 
21
23
  # s3 express session auth
22
24
  autoload :ExpressCredentials, 'aws-sdk-s3/express_credentials'
23
25
  autoload :ExpressCredentialsProvider, 'aws-sdk-s3/express_credentials_provider'
24
26
 
25
27
  # s3 access grants auth
26
-
27
28
  autoload :AccessGrantsCredentials, 'aws-sdk-s3/access_grants_credentials'
28
29
  autoload :AccessGrantsCredentialsProvider, 'aws-sdk-s3/access_grants_credentials_provider'
29
30
  end
@@ -1,9 +1,8 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'pathname'
4
- require 'thread'
4
+ require 'securerandom'
5
5
  require 'set'
6
- require 'tmpdir'
7
6
 
8
7
  module Aws
9
8
  module S3
@@ -12,7 +11,6 @@ module Aws
12
11
 
13
12
  MIN_CHUNK_SIZE = 5 * 1024 * 1024
14
13
  MAX_PARTS = 10_000
15
- THREAD_COUNT = 10
16
14
 
17
15
  def initialize(options = {})
18
16
  @client = options[:client] || Client.new
@@ -22,18 +20,18 @@ module Aws
22
20
  attr_reader :client
23
21
 
24
22
  def download(destination, options = {})
25
- @path = destination
26
- @mode = options[:mode] || 'auto'
27
- @thread_count = options[:thread_count] || THREAD_COUNT
28
- @chunk_size = options[:chunk_size]
29
- @params = {
30
- bucket: options[:bucket],
31
- key: options[:key]
32
- }
33
- @params[:version_id] = options[:version_id] if options[:version_id]
34
- @on_checksum_validated = options[:on_checksum_validated]
35
- @progress_callback = options[:progress_callback]
36
-
23
+ valid_types = [String, Pathname, File, Tempfile]
24
+ unless valid_types.include?(destination.class)
25
+ raise ArgumentError, "Invalid destination, expected #{valid_types.join(', ')} but got: #{destination.class}"
26
+ end
27
+
28
+ @destination = destination
29
+ @mode = options.delete(:mode) || 'auto'
30
+ @thread_count = options.delete(:thread_count) || 10
31
+ @chunk_size = options.delete(:chunk_size)
32
+ @on_checksum_validated = options.delete(:on_checksum_validated)
33
+ @progress_callback = options.delete(:progress_callback)
34
+ @params = options
37
35
  validate!
38
36
 
39
37
  Aws::Plugins::UserAgent.metric('S3_TRANSFER') do
@@ -41,32 +39,31 @@ module Aws
41
39
  when 'auto' then multipart_download
42
40
  when 'single_request' then single_request
43
41
  when 'get_range'
44
- if @chunk_size
45
- resp = @client.head_object(@params)
46
- multithreaded_get_by_ranges(resp.content_length, resp.etag)
47
- else
48
- msg = 'In :get_range mode, :chunk_size must be provided'
49
- raise ArgumentError, msg
50
- end
42
+ raise ArgumentError, 'In get_range mode, :chunk_size must be provided' unless @chunk_size
43
+
44
+ resp = @client.head_object(@params)
45
+ multithreaded_get_by_ranges(resp.content_length, resp.etag)
51
46
  else
52
- msg = "Invalid mode #{@mode} provided, "\
53
- 'mode should be :single_request, :get_range or :auto'
54
- raise ArgumentError, msg
47
+ raise ArgumentError, "Invalid mode #{@mode} provided, :mode should be single_request, get_range or auto"
55
48
  end
56
49
  end
50
+ File.rename(@temp_path, @destination) if @temp_path
51
+ ensure
52
+ File.delete(@temp_path) if @temp_path && File.exist?(@temp_path)
57
53
  end
58
54
 
59
55
  private
60
56
 
61
57
  def validate!
62
- if @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
63
- raise ArgumentError, 'on_checksum_validated must be callable'
64
- end
58
+ return unless @on_checksum_validated && !@on_checksum_validated.respond_to?(:call)
59
+
60
+ raise ArgumentError, ':on_checksum_validated must be callable'
65
61
  end
66
62
 
67
63
  def multipart_download
68
64
  resp = @client.head_object(@params.merge(part_number: 1))
69
65
  count = resp.parts_count
66
+
70
67
  if count.nil? || count <= 1
71
68
  if resp.content_length <= MIN_CHUNK_SIZE
72
69
  single_request
@@ -74,8 +71,8 @@ module Aws
74
71
  multithreaded_get_by_ranges(resp.content_length, resp.etag)
75
72
  end
76
73
  else
77
- # partNumber is an option
78
- resp = @client.head_object(@params)
74
+ # covers cases when given object is not uploaded via UploadPart API
75
+ resp = @client.head_object(@params) # partNumber is an option
79
76
  if resp.content_length <= MIN_CHUNK_SIZE
80
77
  single_request
81
78
  else
@@ -86,7 +83,7 @@ module Aws
86
83
 
87
84
  def compute_mode(file_size, count, etag)
88
85
  chunk_size = compute_chunk(file_size)
89
- part_size = (file_size.to_f / count.to_f).ceil
86
+ part_size = (file_size.to_f / count).ceil
90
87
  if chunk_size < part_size
91
88
  multithreaded_get_by_ranges(file_size, etag)
92
89
  else
@@ -94,32 +91,10 @@ module Aws
94
91
  end
95
92
  end
96
93
 
97
- def construct_chunks(file_size)
98
- offset = 0
99
- default_chunk_size = compute_chunk(file_size)
100
- chunks = []
101
- while offset < file_size
102
- progress = offset + default_chunk_size
103
- progress = file_size if progress > file_size
104
- chunks << "bytes=#{offset}-#{progress - 1}"
105
- offset = progress
106
- end
107
- chunks
108
- end
109
-
110
94
  def compute_chunk(file_size)
111
- if @chunk_size && @chunk_size > file_size
112
- raise ArgumentError, ":chunk_size shouldn't exceed total file size."
113
- else
114
- @chunk_size || [
115
- (file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE
116
- ].max.to_i
117
- end
118
- end
95
+ raise ArgumentError, ":chunk_size shouldn't exceed total file size." if @chunk_size && @chunk_size > file_size
119
96
 
120
- def batches(chunks, mode)
121
- chunks = (1..chunks) if mode.eql? 'part_number'
122
- chunks.each_slice(@thread_count).to_a
97
+ @chunk_size || [(file_size.to_f / MAX_PARTS).ceil, MIN_CHUNK_SIZE].max.to_i
123
98
  end
124
99
 
125
100
  def multithreaded_get_by_ranges(file_size, etag)
@@ -130,12 +105,8 @@ module Aws
130
105
  while offset < file_size
131
106
  progress = offset + default_chunk_size
132
107
  progress = file_size if progress > file_size
133
- range = "bytes=#{offset}-#{progress - 1}"
134
- chunks << Part.new(
135
- part_number: part_number,
136
- size: (progress-offset),
137
- params: @params.merge(range: range, if_match: etag)
138
- )
108
+ params = @params.merge(range: "bytes=#{offset}-#{progress - 1}", if_match: etag)
109
+ chunks << Part.new(part_number: part_number, size: (progress - offset), params: params)
139
110
  part_number += 1
140
111
  offset = progress
141
112
  end
@@ -152,10 +123,13 @@ module Aws
152
123
  def download_in_threads(pending, total_size)
153
124
  threads = []
154
125
  progress = MultipartProgress.new(pending, total_size, @progress_callback) if @progress_callback
126
+ unless [File, Tempfile].include?(@destination.class)
127
+ @temp_path = "#{@destination}.s3tmp.#{SecureRandom.alphanumeric(8)}"
128
+ end
155
129
  @thread_count.times do
156
130
  thread = Thread.new do
157
131
  begin
158
- while part = pending.shift
132
+ while (part = pending.shift)
159
133
  if progress
160
134
  part.params[:on_chunk_received] =
161
135
  proc do |_chunk, bytes, total|
@@ -163,16 +137,17 @@ module Aws
163
137
  end
164
138
  end
165
139
  resp = @client.get_object(part.params)
166
- write(resp)
140
+ range = extract_range(resp.content_range)
141
+ validate_range(range, part.params[:range]) if part.params[:range]
142
+ write(resp.body, range)
167
143
  if @on_checksum_validated && resp.checksum_validated
168
144
  @on_checksum_validated.call(resp.checksum_validated, resp)
169
145
  end
170
146
  end
171
147
  nil
172
- rescue => error
173
- # keep other threads from downloading other parts
174
- pending.clear!
175
- raise error
148
+ rescue StandardError => e
149
+ pending.clear! # keep other threads from downloading other parts
150
+ raise e
176
151
  end
177
152
  end
178
153
  threads << thread
@@ -180,21 +155,28 @@ module Aws
180
155
  threads.map(&:value).compact
181
156
  end
182
157
 
183
- def write(resp)
184
- range, _ = resp.content_range.split(' ').last.split('/')
185
- head, _ = range.split('-').map {|s| s.to_i}
186
- File.write(@path, resp.body.read, head)
158
+ def extract_range(value)
159
+ value.match(%r{bytes (?<range>\d+-\d+)/\d+})[:range]
160
+ end
161
+
162
+ def validate_range(actual, expected)
163
+ return if actual == expected.match(/bytes=(?<range>\d+-\d+)/)[:range]
164
+
165
+ raise MultipartDownloadError, "multipart download failed: expected range of #{expected} but got #{actual}"
166
+ end
167
+
168
+ def write(body, range)
169
+ path = @temp_path || @destination
170
+ File.write(path, body.read, range.split('-').first.to_i)
187
171
  end
188
172
 
189
173
  def single_request
190
- params = @params.merge(response_target: @path)
174
+ params = @params.merge(response_target: @destination)
191
175
  params[:on_chunk_received] = single_part_progress if @progress_callback
192
176
  resp = @client.get_object(params)
193
-
194
177
  return resp unless @on_checksum_validated
195
178
 
196
179
  @on_checksum_validated.call(resp.checksum_validated, resp) if resp.checksum_validated
197
-
198
180
  resp
199
181
  end
200
182
 
@@ -204,6 +186,7 @@ module Aws
204
186
  end
205
187
  end
206
188
 
189
+ # @api private
207
190
  class Part < Struct.new(:part_number, :size, :params)
208
191
  include Aws::Structure
209
192
  end