aws-sdk-s3 1.132.1 → 1.133.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3e9c124873324a9bbaa9f9486830a64940d0b0eef231e64a411ff9b324c740be
-  data.tar.gz: cf26234ca89937e9aa39e44be5ad26b85902ad2eca5f1dbfa9fccf32a93e917e
+  metadata.gz: 04f11cfde483787571492064565bb589b16323f4bb7f33f5a7507b993f6cead3
+  data.tar.gz: 1dd900a34a20b97fb82c41407ca12c40d33397fb14e29678e0f7e3b0a1e5b77c
 SHA512:
-  metadata.gz: 534772057f8571cdb4ce8e6ba47350b672723cc37f654cd5befa1e426c59eb12b7ddb51620ed20dc513c7e4b16dfab54507c261ad8124ca1abe841a1573e1b9f
-  data.tar.gz: ab4ebcde9d217b7e14c7f94696793ece9a2c5c74921e5e087f156585ffd76d9536776054a1a76e9b5965576d41bd9a1d0efc20fddda1aa472272f15d71e48322
+  metadata.gz: 745b68d37b0845463f3c2a1cfdb29e8833bd0d723e31a1071c1f3936f095e4c735c909573c8df49f077b3f56dfa13f9e45552c68d7e5a5c0718b67d3f08418e2
+  data.tar.gz: f995dde0e5628863ab664b36a4db27a23e8d68827aab268fecacfa217a104f1bb3cfd67e84bf6729e52544d9015fd3c64765a53fe0758647e5904c1f104235e1

data/CHANGELOG.md

@@ -1,6 +1,13 @@
 Unreleased Changes
 ------------------
 
+1.133.0 (2023-08-22)
+------------------
+
+* Feature - Code Generated Changes, see `./build_tools` or `aws-sdk-core`'s CHANGELOG.md for details.
+
+* Feature - Add support for `progress_callback` in `Object#download_file` and improve multi-threaded performance #(2901).
+
 1.132.1 (2023-08-09)
 ------------------
 

data/VERSION

@@ -1 +1 @@
-1.132.1
+1.133.0

data/lib/aws-sdk-s3/client.rb

@@ -15653,7 +15653,7 @@ module Aws::S3
         params: params,
         config: config)
       context[:gem_name] = 'aws-sdk-s3'
-      context[:gem_version] = '1.132.1'
+      context[:gem_version] = '1.133.0'
       Seahorse::Client::Request.new(handlers, context)
     end
 

data/lib/aws-sdk-s3/customizations/object.rb

@@ -353,6 +353,10 @@ module Aws
       #     obj.upload_stream do |write_stream|
       #       IO.copy_stream(STDIN, write_stream)
       #     end
+      # @param [Hash] options
+      #   Additional options for {Client#create_multipart_upload},
+      #   {Client#complete_multipart_upload},
+      #   and {Client#upload_part} can be provided.
       #
       # @option options [Integer] :thread_count (10) The number of parallel
       #   multipart uploads
@@ -375,6 +379,9 @@ module Aws
       # @return [Boolean] Returns `true` when the object is uploaded
       #   without any errors.
       #
+      # @see Client#create_multipart_upload
+      # @see Client#complete_multipart_upload
+      # @see Client#upload_part
       def upload_stream(options = {}, &block)
         uploading_options = options.dup
         uploader = MultipartStreamUploader.new(
@@ -427,6 +434,13 @@ module Aws
       #   using an open Tempfile, rewind it before uploading or else the object
       #   will be empty.
       #
+      # @param [Hash] options
+      #   Additional options for {Client#put_object}
+      #   when file sizes below the multipart threshold. For files larger than
+      #   the multipart threshold, options for {Client#create_multipart_upload},
+      #   {Client#complete_multipart_upload},
+      #   and {Client#upload_part} can be provided.
+      #
       # @option options [Integer] :multipart_threshold (104857600) Files larger
       #   than or equal to `:multipart_threshold` are uploaded using the S3
       #   multipart APIs.
@@ -448,6 +462,11 @@ module Aws
       #
       # @return [Boolean] Returns `true` when the object is uploaded
       #   without any errors.
+      #
+      # @see Client#put_object
+      # @see Client#create_multipart_upload
+      # @see Client#complete_multipart_upload
+      # @see Client#upload_part
       def upload_file(source, options = {})
         uploading_options = options.dup
         uploader = FileUploader.new(
@@ -475,8 +494,21 @@ module Aws
       #     # and the parts are downloaded in parallel
       #     obj.download_file('/path/to/very_large_file')
       #
+      # You can provide a callback to monitor progress of the download:
+      #
+      #     # bytes and part_sizes are each an array with 1 entry per part
+      #     # part_sizes may not be known until the first bytes are retrieved
+      #     progress = Proc.new do |bytes, part_sizes, file_size|
+      #       puts bytes.map.with_index { |b, i| "Part #{i+1}: #{b} / #{part_sizes[i]}"}.join(' ') + "Total: #{100.0 * bytes.sum / file_size}%" }
+      #     end
+      #     obj.download_file('/path/to/file', progress_callback: progress)
+      #
       # @param [String] destination Where to download the file to.
       #
+      # @param [Hash] options
+      #   Additional options for {Client#get_object} and #{Client#head_object}
+      #   may be provided.
+      #
       # @option options [String] mode `auto`, `single_request`, `get_range`
       #  `single_request` mode forces only 1 GET request is made in download,
       #  `get_range` mode allows `chunk_size` parameter to configured in
@@ -505,8 +537,18 @@ module Aws
       #   response.  For multipart downloads, this will be called for each
       #   part that is downloaded and validated.
       #
+      # @option options [Proc] :progress_callback
+      #   A Proc that will be called when each chunk of the download is received.
+      #   It will be invoked with [bytes_read], [part_sizes], file_size.
+      #   When the object is downloaded as parts (rather than by ranges), the
+      #   part_sizes will not be known ahead of time and will be nil in the
+      #   callback until the first bytes in the part are received.
+      #
       # @return [Boolean] Returns `true` when the file is downloaded without
       #   any errors.
+      #
+      # @see Client#get_object
+      # @see Client#head_object
       def download_file(destination, options = {})
         downloader = FileDownloader.new(client: client)
         Aws::Plugins::UserAgent.feature('resource') do

data/lib/aws-sdk-s3/file_downloader.rb

@@ -40,6 +40,8 @@ module Aws
         end
         @on_checksum_validated = options[:on_checksum_validated]
 
+        @progress_callback = options[:progress_callback]
+
         validate!
 
         Aws::Plugins::UserAgent.feature('s3-transfer') do
@@ -49,7 +51,7 @@ module Aws
           when 'get_range'
             if @chunk_size
               resp = @client.head_object(@params)
-              multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+              multithreaded_get_by_ranges(resp.content_length)
             else
               msg = 'In :get_range mode, :chunk_size must be provided'
               raise ArgumentError, msg
@@ -82,7 +84,7 @@ module Aws
           if resp.content_length <= MIN_CHUNK_SIZE
             single_request
           else
-            multithreaded_get_by_ranges(construct_chunks(resp.content_length))
+            multithreaded_get_by_ranges(resp.content_length)
           end
         else
           # partNumber is an option
@@ -99,9 +101,9 @@ module Aws
         chunk_size = compute_chunk(file_size)
         part_size = (file_size.to_f / count.to_f).ceil
         if chunk_size < part_size
-          multithreaded_get_by_ranges(construct_chunks(file_size))
+          multithreaded_get_by_ranges(file_size)
         else
-          multithreaded_get_by_parts(count)
+          multithreaded_get_by_parts(count, file_size)
         end
       end
 
@@ -133,30 +135,65 @@ module Aws
         chunks.each_slice(@thread_count).to_a
       end
 
-      def multithreaded_get_by_ranges(chunks)
-        thread_batches(chunks, 'range')
+      def multithreaded_get_by_ranges(file_size)
+        offset = 0
+        default_chunk_size = compute_chunk(file_size)
+        chunks = []
+        part_number = 1 # parts start at 1
+        while offset < file_size
+          progress = offset + default_chunk_size
+          progress = file_size if progress > file_size
+          range = "bytes=#{offset}-#{progress - 1}"
+          chunks << Part.new(
+            part_number: part_number,
+            size: (progress-offset),
+            params: @params.merge(range: range)
+          )
+          part_number += 1
+          offset = progress
+        end
+        download_in_threads(PartList.new(chunks), file_size)
       end
 
-      def multithreaded_get_by_parts(parts)
-        thread_batches(parts, 'part_number')
+      def multithreaded_get_by_parts(n_parts, total_size)
+        parts = (1..n_parts).map do |part|
+          Part.new(part_number: part, params: @params.merge(part_number: part))
+        end
+        download_in_threads(PartList.new(parts), total_size)
       end
 
-      def thread_batches(chunks, param)
-        batches(chunks, param).each do |batch|
-          threads = []
-          batch.each do |chunk|
-            threads << Thread.new do
-              resp = @client.get_object(
-                @params.merge(param.to_sym => chunk)
-              )
-              write(resp)
-              if @on_checksum_validated && resp.checksum_validated
-                @on_checksum_validated.call(resp.checksum_validated, resp)
+      def download_in_threads(pending, total_size)
+        threads = []
+        if @progress_callback
+          progress = MultipartProgress.new(pending, total_size, @progress_callback)
+        end
+        @thread_count.times do
+          thread = Thread.new do
+            begin
+              while part = pending.shift
+                if progress
+                  part.params[:on_chunk_received] =
+                    proc do |_chunk, bytes, total|
+                      progress.call(part.part_number, bytes, total)
+                    end
+                end
+                resp = @client.get_object(part.params)
+                write(resp)
+                if @on_checksum_validated && resp.checksum_validated
+                  @on_checksum_validated.call(resp.checksum_validated, resp)
+                end
               end
+              nil
+            rescue => error
+              # keep other threads from downloading other parts
+              pending.clear!
+              raise error
             end
           end
-          threads.each(&:join)
+          thread.abort_on_exception = true
+          threads << thread
         end
+        threads.map(&:value).compact
       end
 
       def write(resp)
@@ -166,9 +203,9 @@ module Aws
       end
 
       def single_request
-        resp = @client.get_object(
-          @params.merge(response_target: @path)
-        )
+        params = @params.merge(response_target: @path)
+        params[:on_chunk_received] = single_part_progress if @progress_callback
+        resp = @client.get_object(params)
 
         return resp unless @on_checksum_validated
 
@@ -178,6 +215,59 @@ module Aws
 
         resp
       end
+
+      def single_part_progress
+        proc do |_chunk, bytes_read, total_size|
+          @progress_callback.call([bytes_read], [total_size], total_size)
+        end
+      end
+
+      class Part < Struct.new(:part_number, :size, :params)
+        include Aws::Structure
+      end
+
+      # @api private
+      class PartList
+        include Enumerable
+        def initialize(parts = [])
+          @parts = parts
+          @mutex = Mutex.new
+        end
+
+        def shift
+          @mutex.synchronize { @parts.shift }
+        end
+
+        def size
+          @mutex.synchronize { @parts.size }
+        end
+
+        def clear!
+          @mutex.synchronize { @parts.clear }
+        end
+
+        def each(&block)
+          @mutex.synchronize { @parts.each(&block) }
+        end
+      end
+
+      # @api private
+      class  MultipartProgress
+        def initialize(parts, total_size, progress_callback)
+          @bytes_received = Array.new(parts.size, 0)
+          @part_sizes = parts.map(&:size)
+          @total_size = total_size
+          @progress_callback = progress_callback
+        end
+
+        def call(part_number, bytes_received, total)
+          # part numbers start at 1
+          @bytes_received[part_number - 1] = bytes_received
+          # part size may not be known until we get the first response
+          @part_sizes[part_number - 1] ||= total
+          @progress_callback.call(@bytes_received, @part_sizes, @total_size)
+        end
+      end
     end
   end
 end

data/lib/aws-sdk-s3.rb

@@ -73,6 +73,6 @@ require_relative 'aws-sdk-s3/event_streams'
 # @!group service
 module Aws::S3
 
-  GEM_VERSION = '1.132.1'
+  GEM_VERSION = '1.133.0'
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-sdk-s3
 version: !ruby/object:Gem::Version
-  version: 1.132.1
+  version: 1.133.0
 platform: ruby
 authors:
 - Amazon Web Services
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-09 00:00:00.000000000 Z
+date: 2023-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-kms
@@ -47,7 +47,7 @@ dependencies:
         version: '3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.179.0
+        version: 3.181.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -57,7 +57,7 @@ dependencies:
         version: '3'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 3.179.0
+        version: 3.181.0
 description: Official AWS Ruby gem for Amazon Simple Storage Service (Amazon S3).
   This gem is part of the AWS SDK for Ruby.
 email: